[ARVADOS] updated: fbc6c71ef67461cde5ba914e61f3f3b8740a4045
Git user
git at public.curoverse.com
Fri Feb 10 13:51:49 EST 2017
Summary of changes:
sdk/go/manifest/manifest.go | 11 +--
services/crunch-run/crunchrun.go | 36 +++++----
services/crunch-run/crunchrun_test.go | 144 ++++++++++++----------------------
3 files changed, 80 insertions(+), 111 deletions(-)
via fbc6c71ef67461cde5ba914e61f3f3b8740a4045 (commit)
via 397a747639fe360c18f3aa79846fef530b263184 (commit)
from ad464d416de6996a41d0c752124e0d201de0d3fc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fbc6c71ef67461cde5ba914e61f3f3b8740a4045
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 10 13:51:42 2017 -0500
9397: Fix bind behavior to include Path. Update tests to check container binds
as well as output manifest.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f73ec73..5f4de14 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -317,7 +317,21 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if mnt.Writable {
return fmt.Errorf("Can never write to a collection specified by portable data hash")
}
+ idx := strings.Index(mnt.PortableDataHash, "/")
+ if idx > 0 {
+ mnt.Path = path.Clean(mnt.PortableDataHash[idx:])
+ mnt.PortableDataHash = mnt.PortableDataHash[0:idx]
+ runner.Container.Mounts[bind] = mnt
+ }
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.PortableDataHash)
+ if mnt.Path != "" && mnt.Path != "." {
+ if strings.HasPrefix(mnt.Path, "./") {
+ mnt.Path = mnt.Path[2:]
+ } else if strings.HasPrefix(mnt.Path, "/") {
+ mnt.Path = mnt.Path[1:]
+ }
+ src += "/" + mnt.Path
+ }
} else {
src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount)
arvMountCmd = append(arvMountCmd, "--mount-tmp")
@@ -674,20 +688,10 @@ func (runner *ContainerRunner) CaptureOutput() error {
continue
}
- if strings.HasPrefix(bindSuffix, "/") == false {
- bindSuffix = "/" + bindSuffix
- }
-
if mnt.ExcludeFromOutput == true {
continue
}
- idx := strings.Index(mnt.PortableDataHash, "/")
- if idx > 0 {
- mnt.Path = mnt.PortableDataHash[idx:]
- mnt.PortableDataHash = mnt.PortableDataHash[0:idx]
- }
-
// append to manifest_text
m, err := runner.getCollectionManifestForPath(mnt, bindSuffix)
if err != nil {
@@ -699,7 +703,8 @@ func (runner *ContainerRunner) CaptureOutput() error {
// Save output
var response arvados.Collection
- manifestText := manifest.Manifest{Text: manifestText}.NormalizedText()
+ manifest := manifest.Manifest{Text: manifestText}
+ manifestText = manifest.Extract(".", ".").Text
err = runner.ArvClient.Create("collections",
arvadosclient.Dict{
"collection": arvadosclient.Dict{
@@ -747,9 +752,12 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
return "", nil
}
- manifest := manifest.Manifest{Text: collection.ManifestText}
- manifestText := manifest.ManifestTextForPath(mnt.Path, bindSuffix)
- return manifestText, nil
+ mft := manifest.Manifest{Text: collection.ManifestText}
+ extracted := mft.Extract(mnt.Path, bindSuffix)
+ if extracted.Err != nil {
+ return "", fmt.Errorf("Error parsing manifest for %v: %v", mnt.PortableDataHash, extracted.Err.Error())
+ }
+ return extracted.Text, nil
}
func (runner *ContainerRunner) loadDiscoveryVars() {
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index d4b7c1b..d1d2cee 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -57,10 +57,10 @@ var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea
var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d at 5693216f 0:46:md5sum.txt\n"
var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
-var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
+var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
-var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 5676:10:subdir1/file1_in_subdir1.txt 5686:18:subdir1/file2_in_subdir1.txt\n"
+var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
@@ -536,7 +536,7 @@ func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
// Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
// dress rehearsal of the Run() function, starting from a JSON container record.
-func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner) {
+func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
rec := arvados.Container{}
err := json.Unmarshal([]byte(record), &rec)
c.Check(err, IsNil)
@@ -552,6 +552,22 @@ func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDoc
am := &ArvMountCmdLine{}
cr.RunArvMount = am.ArvMountTest
+ realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
+ c.Assert(err, IsNil)
+ defer os.RemoveAll(realTemp)
+
+ tempcount := 0
+ cr.MkTempDir = func(_ string, prefix string) (string, error) {
+ tempcount++
+ d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
+ err := os.Mkdir(d, os.ModePerm)
+ if err != nil && strings.Contains(err.Error(), ": file exists") {
+ // Test case must have pre-populated the tempdir
+ err = nil
+ }
+ return d, err
+ }
+
if extraMounts != nil && len(extraMounts) > 0 {
err := cr.SetupArvMountPoint("keep")
c.Check(err, IsNil)
@@ -578,7 +594,7 @@ func FullRunHelper(c *C, record string, extraMounts []string, fn func(t *TestDoc
}
func (s *TestSuite) TestFullRunHello(c *C) {
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["echo", "hello world"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -600,7 +616,7 @@ func (s *TestSuite) TestFullRunHello(c *C) {
}
func (s *TestSuite) TestCrunchstat(c *C) {
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["sleep", "1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -633,7 +649,7 @@ func (s *TestSuite) TestCrunchstat(c *C) {
}
func (s *TestSuite) TestFullRunStderr(c *C) {
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -659,7 +675,7 @@ func (s *TestSuite) TestFullRunStderr(c *C) {
}
func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["pwd"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": ".",
@@ -681,7 +697,7 @@ func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
}
func (s *TestSuite) TestFullRunSetCwd(c *C) {
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["pwd"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -755,7 +771,7 @@ func (s *TestSuite) TestCancel(c *C) {
}
func (s *TestSuite) TestFullRunSetEnv(c *C) {
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -1022,7 +1038,7 @@ func (s *TestSuite) TestStdout(c *C) {
"runtime_constraints": {}
}`
- api, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
+ api, _, _ := FullRunHelper(c, helperRecord, nil, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
@@ -1085,7 +1101,7 @@ func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
func (s *TestSuite) TestFullRunWithAPI(c *C) {
os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
defer os.Unsetenv("ARVADOS_API_HOST")
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -1109,7 +1125,7 @@ func (s *TestSuite) TestFullRunWithAPI(c *C) {
func (s *TestSuite) TestFullRunSetOutput(c *C) {
os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
defer os.Unsetenv("ARVADOS_API_HOST")
- api, _ := FullRunHelper(c, `{
+ api, _, _ := FullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
"cwd": "/bin",
@@ -1150,7 +1166,7 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
- api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
@@ -1169,64 +1185,10 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"/tmp": {"kind": "tmp"},
- "/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/"},
"/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
"/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
"/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
- "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
- "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
- },
- "output_path": "/tmp",
- "priority": 1,
- "runtime_constraints": {}
- }`
-
- extraMounts := []string{"a0def87f80dd594d4675809e83bd4f15+367"}
-
- api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
- t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
- t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{ExitCode: 0}
- })
-
- c.Check(api.CalledWith("container.exit_code", 0), NotNil)
- c.Check(api.CalledWith("container.state", "Complete"), NotNil)
- for _, v := range api.Content {
- if v["collection"] != nil {
- collection := v["collection"].(arvadosclient.Dict)
- if strings.Index(collection["name"].(string), "output") == 0 {
- manifest := collection["manifest_text"].(string)
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
-
- origManifestWithDotReplacedAsFoo := strings.Replace(normalizedManifestWithSubdirs, "./", "./foo/", -1)
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar"))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:sub1file2"))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:18:sub2file2"))
- }
- }
- }
-}
-
-func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
- helperRecord := `{
- "command": ["/bin/sh", "-c", "echo $FROBIZ"],
- "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
- "cwd": "/bin",
- "environment": {"FROBIZ": "bilbo"},
- "mounts": {
- "/tmp": {"kind": "tmp"},
- "/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"},
- "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt"},
- "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/"},
- "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
- "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt"},
+ "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
"stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
},
"output_path": "/tmp",
@@ -1235,18 +1197,24 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
}`
extraMounts := []string{
- "a0def87f80dd594d4675809e83bd4f15+367",
"a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
"a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
"a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
}
- api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
})
+ c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
+ realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
+ realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
+ realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
+ realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
+ })
+
c.Check(api.CalledWith("container.exit_code", 0), NotNil)
c.Check(api.CalledWith("container.state", "Complete"), NotNil)
for _, v := range api.Content {
@@ -1255,18 +1223,12 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
if strings.Index(collection["name"].(string), "output") == 0 {
manifest := collection["manifest_text"].(string)
- c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
-
- origManifestWithDotReplacedAsFoo := strings.Replace(normalizedManifestWithSubdirs, "./", "./foo/", -1)
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar"))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:sub1file2"))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:18:sub2file2"))
+ c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar 9:18:sub1file2
+./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:18:sub2file2
+./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
+./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
+`)
}
}
}
@@ -1280,8 +1242,7 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"/tmp": {"kind": "tmp"},
- "/tmp/foo": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path":"/"},
- "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path":"subdir1/file2_in_subdir1.txt"},
+ "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
"stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
},
"output_path": "/tmp",
@@ -1289,9 +1250,11 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
"runtime_constraints": {}
}`
- extraMounts := []string{"b0def87f80dd594d4675809e83bd4f15+367"}
+ extraMounts := []string{
+ "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
+ }
- api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ api, _, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
t.logWriter.Close()
t.finish <- dockerclient.WaitResult{ExitCode: 0}
@@ -1305,12 +1268,9 @@ func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(
if strings.Index(collection["name"].(string), "output") == 0 {
manifest := collection["manifest_text"].(string)
- c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
-
- origManifestWithDotReplacedAsFoo := strings.Replace(denormalizedManifestWithSubdirs, "./", "./foo/", -1)
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
-
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 5686:18:subdir1/file2_in_subdir1.txt"))
+ c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
+./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 10:17:bar
+`)
}
}
}
commit 397a747639fe360c18f3aa79846fef530b263184
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 10 13:40:31 2017 -0500
9397: Adjust stream normalization to use just digest (no hints) to de-duplicate
block locators.
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index cb60560..9038960 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -356,16 +356,16 @@ func (stream segmentedStream) normalizedText(name string) string {
stream_tokens := []string{EscapeName(name)}
- blocks := make(map[string]int64)
+ blocks := make(map[blockdigest.BlockDigest]int64)
var streamoffset int64
// Go through each file and add each referenced block exactly once.
for _, streamfile := range sortedfiles {
for _, segment := range stream[streamfile] {
- if _, ok := blocks[segment.Locator]; !ok {
+ b, _ := ParseBlockLocator(segment.Locator)
+ if _, ok := blocks[b.Digest]; !ok {
stream_tokens = append(stream_tokens, segment.Locator)
- blocks[segment.Locator] = streamoffset
- b, _ := ParseBlockLocator(segment.Locator)
+ blocks[b.Digest] = streamoffset
streamoffset += int64(b.Size)
}
}
@@ -382,7 +382,8 @@ func (stream segmentedStream) normalizedText(name string) string {
fout := EscapeName(streamfile)
for _, segment := range stream[streamfile] {
// Collapse adjacent segments
- streamoffset = blocks[segment.Locator] + int64(segment.Offset)
+ b, _ := ParseBlockLocator(segment.Locator)
+ streamoffset = blocks[b.Digest] + int64(segment.Offset)
if span_start == -1 {
span_start = streamoffset
span_end = streamoffset + int64(segment.Len)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list