[ARVADOS] updated: 8ee173e80b4954159490c8f9327dab07a4bffcf3
Git user
git at public.curoverse.com
Mon Feb 6 12:30:36 EST 2017
Summary of changes:
sdk/go/manifest/manifest.go | 2 ++
sdk/go/manifest/manifest_test.go | 6 +++++
services/crunch-run/crunchrun.go | 30 ++++++++++++++++++++-
services/crunch-run/crunchrun_test.go | 49 +++++++++++++++++++++++++++++++++++
4 files changed, 86 insertions(+), 1 deletion(-)
via 8ee173e80b4954159490c8f9327dab07a4bffcf3 (commit)
via 837c509a55f56fa653454e7b99e293f8f87ebef2 (commit)
from 8adad21150d3678aad0f88e5fb30a088145478ee (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8ee173e80b4954159490c8f9327dab07a4bffcf3
Author: radhika <radhika at curoverse.com>
Date: Sun Feb 5 22:40:00 2017 -0500
9397: add normalized manifest test
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index 5219a58..a9745ae 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -313,6 +313,8 @@ func (m *Manifest) NormalizedManifestForPath(path string) string {
} else {
if currentSpan[1] == fss.SegPos {
currentSpan[1] += fss.SegLen
+ } else if currentSpan[0]+currentSpan[1] == fss.SegPos {
+ currentSpan[1] = fss.SegPos + fss.SegLen
} else {
manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]+fss.SegLen) + ":" + fss.Name + " "
currentSpan = []uint64{fss.SegPos, fss.SegPos + fss.SegLen}
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 2fe4272..0d58a9e 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -251,3 +251,9 @@ func TestBlockIterWithBadManifest(t *testing.T) {
}
}
}
+
+func TestNormalizeManifest(t *testing.T) {
+ m := Manifest{Text: ". acbd18db4cc2f85cedef654fccc4a4d8+40 0:10:one 10:10:one 20:10:two 30:10:two\n"}
+ normalized := m.NormalizedManifestForPath("")
+ expectEqual(t, normalized, ". acbd18db4cc2f85cedef654fccc4a4d8+40 0:20:one 20:40:two\n")
+}
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 8d29258..6d8ec9c 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -755,12 +755,18 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
manifestText = strings.Replace(collection.ManifestText, "./", "."+bindSuffix+"/", -1)
manifestText = strings.Replace(manifestText, ". ", "."+bindSuffix+" ", -1)
wanted := ""
- for _, token := range strings.Split(manifestText, " ") {
- if strings.Index(token, ":") == -1 {
- wanted += " " + token
- } else if strings.Index(token, ":"+mntPath) >= 0 {
- wanted += " " + token + "\n"
- break
+ for _, stream := range strings.Split(manifestText, "\n") {
+ if strings.Index(stream, mntPath) == -1 {
+ continue
+ }
+
+ for _, token := range strings.Split(manifestText, " ") {
+ if strings.Index(token, ":") == -1 {
+ wanted += " " + token
+ } else if strings.Index(token, ":"+mntPath) >= 0 {
+ wanted += " " + token + "\n"
+ break
+ }
}
}
return wanted, nil
commit 837c509a55f56fa653454e7b99e293f8f87ebef2
Author: radhika <radhika at curoverse.com>
Date: Sun Feb 5 21:03:11 2017 -0500
9397: add handling of denormalized manifest
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 40174b0..8d29258 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -741,9 +741,31 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
outputCollections[mnt.PortableDataHash] = collection
}
- manifest := manifest.Manifest{Text: collection.ManifestText}
+ if collection.ManifestText == "" {
+ runner.CrunchLog.Printf("No manifest text for collection %v", collection.PortableDataHash)
+ return "", nil
+ }
+ manifest := manifest.Manifest{Text: collection.ManifestText}
manifestText := manifest.NormalizedManifestForPath(mnt.Path)
+
+ if manifestText == "" {
+ // It could be denormalized manifest
+ mntPath := strings.Trim(mnt.Path, "/")
+ manifestText = strings.Replace(collection.ManifestText, "./", "."+bindSuffix+"/", -1)
+ manifestText = strings.Replace(manifestText, ". ", "."+bindSuffix+" ", -1)
+ wanted := ""
+ for _, token := range strings.Split(manifestText, " ") {
+ if strings.Index(token, ":") == -1 {
+ wanted += " " + token
+ } else if strings.Index(token, ":"+mntPath) >= 0 {
+ wanted += " " + token + "\n"
+ break
+ }
+ }
+ return wanted, nil
+ }
+
if mnt.Path == "" || mnt.Path == "/" {
// no path specified; return the entire manifest text after making adjustments
manifestText = strings.Replace(manifestText, "./", "."+bindSuffix+"/", -1)
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 8d0322c..d4b7c1b 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -60,6 +60,9 @@ var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
+var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 5676:10:subdir1/file1_in_subdir1.txt 5686:18:subdir1/file2_in_subdir1.txt\n"
+var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
+
var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
@@ -187,6 +190,8 @@ func (client *ArvTestClient) Get(resourceType string, uuid string, parameters ar
output.(*arvados.Collection).ManifestText = otherManifest
} else if uuid == normalizedWithSubdirsPDH {
output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
+ } else if uuid == denormalizedWithSubdirsPDH {
+ output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
}
}
if resourceType == "containers" {
@@ -1266,3 +1271,47 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
}
}
}
+
+func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
+ helperRecord := `{
+ "command": ["/bin/sh", "-c", "echo $FROBIZ"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": "/bin",
+ "environment": {"FROBIZ": "bilbo"},
+ "mounts": {
+ "/tmp": {"kind": "tmp"},
+ "/tmp/foo": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path":"/"},
+ "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path":"subdir1/file2_in_subdir1.txt"},
+ "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`
+
+ extraMounts := []string{"b0def87f80dd594d4675809e83bd4f15+367"}
+
+ api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
+ t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
+ t.logWriter.Close()
+ t.finish <- dockerclient.WaitResult{ExitCode: 0}
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+ for _, v := range api.Content {
+ if v["collection"] != nil {
+ collection := v["collection"].(arvadosclient.Dict)
+ if strings.Index(collection["name"].(string), "output") == 0 {
+ manifest := collection["manifest_text"].(string)
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
+
+ origManifestWithDotReplacedAsFoo := strings.Replace(denormalizedManifestWithSubdirs, "./", "./foo/", -1)
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 5686:18:subdir1/file2_in_subdir1.txt"))
+ }
+ }
+ }
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list