[ARVADOS] created: ad464d416de6996a41d0c752124e0d201de0d3fc
Git user
git at public.curoverse.com
Fri Feb 10 11:04:30 EST 2017
at ad464d416de6996a41d0c752124e0d201de0d3fc (commit)
commit ad464d416de6996a41d0c752124e0d201de0d3fc
Merge: a0195a5 80689aa
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 10 11:04:23 2017 -0500
Merge branch 'master' into origin-9397-prepopulate-output-directory
commit a0195a506af77771434302b96a5e05ecfd3814d8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 9 14:28:26 2017 -0500
9397: Fix collection.go to use blockdigest.LocatorPattern
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index 157ce16..df7c224 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -3,10 +3,9 @@ package arvados
import (
"bufio"
"fmt"
+ "git.curoverse.com/arvados.git/sdk/go/blockdigest"
"strings"
"time"
-
- "git.curoverse.com/arvados.git/sdk/go/manifest"
)
// Collection is an arvados#collection resource.
@@ -46,7 +45,7 @@ func (c *Collection) SizedDigests() ([]SizedDigest, error) {
return nil, fmt.Errorf("Invalid stream (<3 tokens): %q", line)
}
for _, token := range tokens[1:] {
- if !manifest.LocatorPattern.MatchString(token) {
+ if !blockdigest.LocatorPattern.MatchString(token) {
// FIXME: ensure it's a file token
break
}
commit 85c684122fd678dc24035d58236dd5734aed50e5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 9 14:25:19 2017 -0500
9397: Use ManifestTextForPath
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 6d8ec9c..f73ec73 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -699,6 +699,7 @@ func (runner *ContainerRunner) CaptureOutput() error {
// Save output
var response arvados.Collection
+ manifestText := manifest.Manifest{Text: manifestText}.NormalizedText()
err = runner.ArvClient.Create("collections",
arvadosclient.Dict{
"collection": arvadosclient.Dict{
@@ -747,68 +748,7 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
}
manifest := manifest.Manifest{Text: collection.ManifestText}
- manifestText := manifest.NormalizedManifestForPath(mnt.Path)
-
- if manifestText == "" {
- // It could be denormalized manifest
- mntPath := strings.Trim(mnt.Path, "/")
- manifestText = strings.Replace(collection.ManifestText, "./", "."+bindSuffix+"/", -1)
- manifestText = strings.Replace(manifestText, ". ", "."+bindSuffix+" ", -1)
- wanted := ""
- for _, stream := range strings.Split(manifestText, "\n") {
- if strings.Index(stream, mntPath) == -1 {
- continue
- }
-
- for _, token := range strings.Split(manifestText, " ") {
- if strings.Index(token, ":") == -1 {
- wanted += " " + token
- } else if strings.Index(token, ":"+mntPath) >= 0 {
- wanted += " " + token + "\n"
- break
- }
- }
- }
- return wanted, nil
- }
-
- if mnt.Path == "" || mnt.Path == "/" {
- // no path specified; return the entire manifest text after making adjustments
- manifestText = strings.Replace(manifestText, "./", "."+bindSuffix+"/", -1)
- manifestText = strings.Replace(manifestText, ". ", "."+bindSuffix+" ", -1)
- } else {
- // either a single stream or file from a stream is being sought
- bindIdx := strings.LastIndex(bindSuffix, "/")
- var bindSubdir, bindFileName string
- if bindIdx >= 0 {
- bindSubdir = "." + bindSuffix[0:bindIdx]
- bindFileName = bindSuffix[bindIdx+1:]
- }
- mntPath := mnt.Path
- if strings.HasSuffix(mntPath, "/") {
- mntPath = mntPath[0 : len(mntPath)-1]
- }
- pathIdx := strings.LastIndex(mntPath, "/")
- var pathSubdir, pathFileName string
- if pathIdx >= 0 {
- pathSubdir = "." + mntPath[0:pathIdx]
- pathFileName = mntPath[pathIdx+1:]
- }
-
- if strings.Index(manifestText, "."+mntPath+" ") != -1 {
- // path refers to this complete stream
- manifestText = strings.Replace(manifestText, "."+mntPath, "."+bindSuffix, -1)
- } else {
- // look for a matching file in this stream
- manifestText = strings.Replace(manifestText, ":"+pathFileName, ":"+bindFileName, -1)
- manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
- }
- }
-
- if manifestText == "" {
- runner.CrunchLog.Printf("No manifest segment found for bind '%v' with path '%v'", bindSuffix, mnt.Path)
- }
-
+ manifestText := manifest.ManifestTextForPath(mnt.Path, bindSuffix)
return manifestText, nil
}
commit 1c434a9f9ec70d8f23583bc737a516c3ef0eb91d
Merge: 310f7b0 5bf9312
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 9 14:11:53 2017 -0500
Merge branch '9397-go-manifest' into origin-9397-prepopulate-output-directory
commit 310f7b0266f012355e38f3296bc6defe14a3d25f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 9 14:11:35 2017 -0500
9397: Remove conflicting code for easier merge
diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index a9745ae..ff104aa 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -230,103 +230,6 @@ func parseManifestStream(s string) (m ManifestStream) {
return
}
-func (m *Manifest) NormalizeManifest() map[string]ManifestStream {
- streams := make(map[string]ManifestStream)
-
- for stream := range m.StreamIter() {
- ms := streams[stream.StreamName]
-
- if ms.StreamName == "" { // new stream
- streams[stream.StreamName] = stream
- } else {
- ms.Blocks = append(ms.Blocks, stream.Blocks...)
- ms.FileStreamSegments = append(ms.FileStreamSegments, stream.FileStreamSegments...)
- }
- }
-
- return streams
-}
-
-func (m *Manifest) NormalizedManifestForPath(path string) string {
- normalized := m.NormalizeManifest()
-
- var streams []string
- for _, stream := range normalized {
- streams = append(streams, stream.StreamName)
- }
- sort.Strings(streams)
-
- path = strings.Trim(path, "/")
- var subdir, filename string
-
- if path != "" {
- if strings.Index(path, "/") == -1 {
- isStream := false
- for _, v := range streams {
- if v == "./"+path {
- isStream = true
- }
- }
- if isStream {
- subdir = path
- } else {
- filename = path
- }
- } else {
- pathIdx := strings.LastIndex(path, "/")
- if pathIdx >= 0 {
- subdir = path[0:pathIdx]
- filename = path[pathIdx+1:]
- }
- }
- }
-
- manifestForPath := ""
-
- for _, streamName := range streams {
- stream := normalized[streamName]
-
- if subdir != "" && stream.StreamName != "./"+subdir {
- continue
- }
-
- manifestForPath += stream.StreamName + " " + strings.Join(stream.Blocks, " ") + " "
-
- currentName := ""
- currentSpan := []uint64{0, 0}
- for _, fss := range stream.FileStreamSegments {
- if filename != "" && fss.Name != filename {
- continue
- }
-
- if fss.Name != currentName && currentName != "" {
- manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
- }
-
- if fss.Name != currentName {
- currentName = fss.Name
- currentSpan = []uint64{0, 0}
- }
-
- if currentSpan[1] == 0 {
- currentSpan = []uint64{fss.SegPos, fss.SegLen}
- } else {
- if currentSpan[1] == fss.SegPos {
- currentSpan[1] += fss.SegLen
- } else if currentSpan[0]+currentSpan[1] == fss.SegPos {
- currentSpan[1] = fss.SegPos + fss.SegLen
- } else {
- manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]+fss.SegLen) + ":" + fss.Name + " "
- currentSpan = []uint64{fss.SegPos, fss.SegPos + fss.SegLen}
- }
- }
- }
- manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + "\n"
- }
-
- return manifestForPath
-}
-
func (m *Manifest) StreamIter() <-chan ManifestStream {
ch := make(chan ManifestStream)
go func(input string) {
diff --git a/sdk/go/manifest/manifest_test.go b/sdk/go/manifest/manifest_test.go
index 0d58a9e..2fe4272 100644
--- a/sdk/go/manifest/manifest_test.go
+++ b/sdk/go/manifest/manifest_test.go
@@ -251,9 +251,3 @@ func TestBlockIterWithBadManifest(t *testing.T) {
}
}
}
-
-func TestNormalizeManifest(t *testing.T) {
- m := Manifest{Text: ". acbd18db4cc2f85cedef654fccc4a4d8+40 0:10:one 10:10:one 20:10:two 30:10:two\n"}
- normalized := m.NormalizedManifestForPath("")
- expectEqual(t, normalized, ". acbd18db4cc2f85cedef654fccc4a4d8+40 0:20:one 20:40:two\n")
-}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list