[ARVADOS] updated: a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a
Git user
git at public.curoverse.com
Mon Jan 30 17:20:54 EST 2017
Summary of changes:
services/crunch-run/crunchrun.go | 30 ++++++++++--------
services/crunch-run/crunchrun_test.go | 57 ++++++++++++++++++++++++-----------
2 files changed, 58 insertions(+), 29 deletions(-)
via a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a (commit)
from 280621308e41e9cc8b8557d27738f186e795780b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a5e7c0c8bf829a41c4d02b48d5a9a40d6d225c8a
Author: radhika <radhika at curoverse.com>
Date: Mon Jan 30 15:58:28 2017 -0500
9397: more testing
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index f510825..3b31da5 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -673,7 +673,7 @@ func (runner *ContainerRunner) CaptureOutput() error {
bindSuffix := strings.TrimPrefix(bind, runner.Container.OutputPath)
if bindSuffix == bind || len(bindSuffix) <= 0 {
- // either doesn't start with OutputPath or is OutputPath itself
+ // either does not start with OutputPath or is OutputPath itself
continue
}
@@ -730,12 +730,18 @@ func (runner *ContainerRunner) CaptureOutput() error {
// Return the manifest_text fragment corresponding to the specified mnt.Path
// after making any required updates.
// Ex:
-// If mnt.Path is not speficied,
+// If mnt.Path is not specified,
// return the entire manifest_text after replacing any "." with bindSuffix
// If mnt.Path corresponds to one stream,
-// return the manitest_text for that stream after replacing that stream name with bindSuffix
-// Otherwise, check if a filename in any one stream is being sought. Return the manitest_text
-// for that stream after replacing that stream name and file name using bindSuffix components.
+// return the manifest_text for that stream after replacing that stream name with bindSuffix
+// Otherwise, check if a filename in any one stream is being sought. Return the manifest_text
+// for that stream after replacing stream name with bindSuffix minus the last word
+// and the file name with last word of the bindSuffix
+// Allowed path examples:
+// "path":"/"
+// "path":"/subdir1"
+// "path":"/subdir1/subdir2"
+// "path":"/subdir/filename" etc
func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, bindSuffix string) (string, error) {
var collection arvados.Collection
err := runner.ArvClient.Get("collections", mnt.PortableDataHash, nil, &collection)
@@ -744,7 +750,7 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
}
manifestText := ""
- if mnt.Path == "" {
+ if mnt.Path == "" || mnt.Path == "/" {
// no path specified; return the entire manifest text
manifestText = collection.ManifestText
manifestText = strings.Replace(manifestText, "./", "."+bindSuffix+"/", -1)
@@ -754,26 +760,26 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
bindIdx := strings.LastIndex(bindSuffix, "/")
var bindSubdir, bindFileName string
if bindIdx >= 0 {
- bindSubdir = bindSuffix[0:bindIdx]
+ bindSubdir = "." + bindSuffix[0:bindIdx]
bindFileName = bindSuffix[bindIdx+1:]
}
pathIdx := strings.LastIndex(mnt.Path, "/")
var pathSubdir, pathFileName string
if pathIdx >= 0 {
- pathSubdir = mnt.Path[0:pathIdx]
+ pathSubdir = "." + mnt.Path[0:pathIdx]
pathFileName = mnt.Path[pathIdx+1:]
}
streams := strings.Split(collection.ManifestText, "\n")
for _, stream := range streams {
tokens := strings.Split(stream, " ")
if tokens[0] == "."+mnt.Path {
- // path refers to this stream
- adjustedStream := strings.Replace(stream, mnt.Path, bindSuffix, -1)
+ // path refers to this complete stream
+ adjustedStream := strings.Replace(stream, "."+mnt.Path, "."+bindSuffix, -1)
manifestText = adjustedStream + "\n"
break
} else {
// look for a matching file in this stream
- if tokens[0] == "."+pathSubdir {
+ if tokens[0] == pathSubdir {
// path refers to a file in this stream
for _, token := range tokens {
if strings.Index(token, ":"+pathFileName) > 0 {
@@ -786,7 +792,7 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
}
}
manifestText = strings.Trim(manifestText, " ")
- token = strings.Replace(token, pathFileName, bindFileName, -1)
+ token = strings.Replace(token, ":"+pathFileName, ":"+bindFileName, -1)
manifestText += (" " + token + "\n")
manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
break
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index fd9f0cb..8b08fb2 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -40,9 +40,9 @@ type ArvTestClient struct {
Calls int
Content []arvadosclient.Dict
arvados.Container
- Logs map[string]*bytes.Buffer
- WasSetRunning bool
+ Logs map[string]*bytes.Buffer
sync.Mutex
+ WasSetRunning bool
}
type KeepTestClient struct {
@@ -57,7 +57,7 @@ var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea
var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d at 5693216f 0:46:md5sum.txt\n"
var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
-var subdirManifest = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0a18eda6411234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:qr1hi-8i9sb-6tjg516fx3dhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0a18eda6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0a18eda6415544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:19:file2_in_subdir2.txt\n"
+var subdirManifest = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 27:5649:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 0:9:file1_in_subdir2.txt 9:19:file2_in_subdir2.txt\n"
var subdirPDH = "a0def87f80dd594d4675809e83bd4f15+367"
var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
@@ -1148,7 +1148,7 @@ func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C
c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
}
-func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
+func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
@@ -1156,7 +1156,11 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
"environment": {"FROBIZ": "bilbo"},
"mounts": {
"/tmp": {"kind": "tmp"},
- "/tmp/foo": {"kind": "collection", "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54/md5sum.txt"},
+ "/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/"},
+ "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
+ "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
+ "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
+ "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
"stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
},
"output_path": "/tmp",
@@ -1164,7 +1168,7 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
"runtime_constraints": {}
}`
- extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54/md5sum.txt"}
+ extraMounts := []string{"a0def87f80dd594d4675809e83bd4f15+367"}
api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
@@ -1178,15 +1182,26 @@ func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
if v["collection"] != nil {
collection := v["collection"].(arvadosclient.Dict)
if strings.Index(collection["name"].(string), "output") == 0 {
- streams := strings.Split(collection["manifest_text"].(string), "\n")
- c.Check(streams[0], Equals, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out")
- c.Check(streams[1], Equals, ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d at 5693216f 0:46:foo")
+ manifest := collection["manifest_text"].(string)
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out"))
+
+ origManifestWithDotReplacedAsFoo := strings.Replace(subdirManifest, "./", "./foo/", -1)
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar"))
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:sub1file2"))
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:19:sub2file2"))
}
}
}
}
-func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
+func (s *TestSuite) TestStdoutWithMountPointForFileUnderOutputDir(c *C) {
helperRecord := `{
"command": ["/bin/sh", "-c", "echo $FROBIZ"],
"container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
@@ -1195,9 +1210,10 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
"mounts": {
"/tmp": {"kind": "tmp"},
"/tmp/foo": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"},
- "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
- "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
- "/tmp/foo/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
+ "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt"},
+ "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1"},
+ "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
+ "/tmp/foo/bar/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt"},
"stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
},
"output_path": "/tmp",
@@ -1205,7 +1221,12 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
"runtime_constraints": {}
}`
- extraMounts := []string{"a0def87f80dd594d4675809e83bd4f15+367"}
+ extraMounts := []string{
+ "a0def87f80dd594d4675809e83bd4f15+367",
+ "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
+ "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
+ "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
+ }
api, _ := FullRunHelper(c, helperRecord, extraMounts, func(t *TestDockerClient) {
t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
@@ -1226,11 +1247,13 @@ func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
origManifestWithDotReplacedAsFoo := strings.Replace(subdirManifest, "./", "./foo/", -1)
c.Check(-1, Not(Equals), strings.Index(manifest, "./foo"+origManifestWithDotReplacedAsFoo[1:]))
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0a18eda6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890 at 569fa8c3 9:18:bar"))
+
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt"))
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0a18eda6419876543234 at 569fa8c4 9:18:sub1file2"))
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234 at 569fa8c4 9:18:sub1file2"))
- c.Check(-1, Not(Equals), strings.Index(manifest, "./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0a18eda6415544332211 at 569fa8c5 9:19:sub2file2"))
+ c.Check(-1, Not(Equals), strings.Index(manifest, "./foo/bar 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211 at 569fa8c5 9:19:sub2file2"))
}
}
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list