[ARVADOS] updated: 8adad21150d3678aad0f88e5fb30a088145478ee

[Git user]

Summary of changes:
 sdk/go/manifest/manifest.go      | 116 ++++++++++++++++++++++-----------------
 services/crunch-run/crunchrun.go |  17 ++----
 2 files changed, 70 insertions(+), 63 deletions(-)

       via  8adad21150d3678aad0f88e5fb30a088145478ee (commit)
      from  785c967e74a7dab0b29b276162d1d7513ce1cf6b (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8adad21150d3678aad0f88e5fb30a088145478ee
Author: radhika <radhika at curoverse.com>
Date:   Sun Feb 5 19:14:48 2017 -0500

    9397: NormalizeManifest

diff --git a/sdk/go/manifest/manifest.go b/sdk/go/manifest/manifest.go
index 18a6b85..5219a58 100644
--- a/sdk/go/manifest/manifest.go
+++ b/sdk/go/manifest/manifest.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/blockdigest"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 )
@@ -229,19 +230,75 @@ func parseManifestStream(s string) (m ManifestStream) {
 	return
 }
 
-func (m *Manifest) ManifestStreamForPath(path string) string {
+func (m *Manifest) NormalizeManifest() map[string]ManifestStream {
+	streams := make(map[string]ManifestStream)
+
+	for stream := range m.StreamIter() {
+		ms := streams[stream.StreamName]
+
+		if ms.StreamName == "" { // new stream
+			streams[stream.StreamName] = stream
+		} else {
+			ms.Blocks = append(ms.Blocks, stream.Blocks...)
+			ms.FileStreamSegments = append(ms.FileStreamSegments, stream.FileStreamSegments...)
+		}
+	}
+
+	return streams
+}
+
+func (m *Manifest) NormalizedManifestForPath(path string) string {
+	normalized := m.NormalizeManifest()
+
+	var streams []string
+	for _, stream := range normalized {
+		streams = append(streams, stream.StreamName)
+	}
+	sort.Strings(streams)
+
+	path = strings.Trim(path, "/")
+	var subdir, filename string
+
+	if path != "" {
+		if strings.Index(path, "/") == -1 {
+			isStream := false
+			for _, v := range streams {
+				if v == "./"+path {
+					isStream = true
+				}
+			}
+			if isStream {
+				subdir = path
+			} else {
+				filename = path
+			}
+		} else {
+			pathIdx := strings.LastIndex(path, "/")
+			if pathIdx >= 0 {
+				subdir = path[0:pathIdx]
+				filename = path[pathIdx+1:]
+			}
+		}
+	}
+
 	manifestForPath := ""
-	locators := []string{}
-	for ms := range m.StreamIter() {
-		if ms.StreamName != "./"+path {
+
+	for _, streamName := range streams {
+		stream := normalized[streamName]
+
+		if subdir != "" && stream.StreamName != "./"+subdir {
 			continue
 		}
 
-		locators = append(locators, ms.Blocks...)
+		manifestForPath += stream.StreamName + " " + strings.Join(stream.Blocks, " ") + " "
 
 		currentName := ""
 		currentSpan := []uint64{0, 0}
-		for _, fss := range ms.FileStreamSegments {
+		for _, fss := range stream.FileStreamSegments {
+			if filename != "" && fss.Name != filename {
+				continue
+			}
+
 			if fss.Name != currentName && currentName != "" {
 				manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
 			}
@@ -262,14 +319,10 @@ func (m *Manifest) ManifestStreamForPath(path string) string {
 				}
 			}
 		}
-		manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + " "
-	}
-
-	if len(locators) > 0 {
-		return "./" + path + " " + strings.Join(locators, " ") + " " + manifestForPath
+		manifestForPath += fmt.Sprintf("%v", currentSpan[0]) + ":" + fmt.Sprintf("%v", currentSpan[1]) + ":" + currentName + "\n"
 	}
 
-	return ""
+	return manifestForPath
 }
 
 func (m *Manifest) StreamIter() <-chan ManifestStream {
@@ -308,45 +361,6 @@ func (m *Manifest) FileSegmentIterByName(filepath string) <-chan *FileSegment {
 	return ch
 }
 
-func (m *Manifest) FileSegmentForPath(filepath string) string {
-	dir := "."
-	file := filepath
-	if idx := strings.LastIndex(filepath, "/"); idx >= 0 {
-		dir = "./" + filepath[0:idx]
-		file = filepath[idx+1:]
-	}
-
-	var fileSegments []FileSegment
-	for fs := range m.FileSegmentIterByName(filepath) {
-		fileSegments = append(fileSegments, *fs)
-	}
-
-	if len(fileSegments) == 0 {
-		return ""
-	}
-
-	locators := ""
-	manifestForFile := ""
-	currentSpan := []int{0, 0}
-	for _, fs := range fileSegments {
-		locators += fs.Locator + " "
-
-		if currentSpan[1] == 0 {
-			currentSpan = []int{fs.Offset, fs.Len}
-		} else {
-			if currentSpan[1] == fs.Offset {
-				currentSpan[1] += fs.Len
-			} else {
-				manifestForFile += strconv.Itoa(currentSpan[0]) + ":" + strconv.Itoa(currentSpan[1]+fs.Len) + ":" + file + " "
-				currentSpan = []int{fs.Offset, fs.Offset + fs.Len}
-			}
-		}
-	}
-	manifestForFile += strconv.Itoa(currentSpan[0]) + ":" + strconv.Itoa(currentSpan[1]) + ":" + file
-
-	return fmt.Sprintf("%v %v %v", dir, strings.Trim(locators, " "), manifestForFile)
-}
-
 // Blocks may appear multiple times within the same manifest if they
 // are used by multiple files. In that case this Iterator will output
 // the same block multiple times.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 5a228e6..40174b0 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -743,10 +743,9 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
 
 	manifest := manifest.Manifest{Text: collection.ManifestText}
 
-	manifestText := ""
+	manifestText := manifest.NormalizedManifestForPath(mnt.Path)
 	if mnt.Path == "" || mnt.Path == "/" {
 		// no path specified; return the entire manifest text after making adjustments
-		manifestText = collection.ManifestText
 		manifestText = strings.Replace(manifestText, "./", "."+bindSuffix+"/", -1)
 		manifestText = strings.Replace(manifestText, ". ", "."+bindSuffix+" ", -1)
 	} else {
@@ -768,19 +767,13 @@ func (runner *ContainerRunner) getCollectionManifestForPath(mnt arvados.Mount, b
 			pathFileName = mntPath[pathIdx+1:]
 		}
 
-		manifestStreamForPath := manifest.ManifestStreamForPath(mntPath[1:])
-		if manifestStreamForPath != "" {
+		if strings.Index(manifestText, "."+mntPath+" ") != -1 {
 			// path refers to this complete stream
-			adjustedStream := strings.Replace(manifestStreamForPath, "."+mntPath, "."+bindSuffix, -1)
-			manifestText = adjustedStream + "\n"
+			manifestText = strings.Replace(manifestText, "."+mntPath, "."+bindSuffix, -1)
 		} else {
 			// look for a matching file in this stream
-			fs := manifest.FileSegmentForPath(mntPath[1:])
-			if fs != "" {
-				manifestText = strings.Replace(fs, ":"+pathFileName, ":"+bindFileName, -1)
-				manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
-				manifestText += "\n"
-			}
+			manifestText = strings.Replace(manifestText, ":"+pathFileName, ":"+bindFileName, -1)
+			manifestText = strings.Replace(manifestText, pathSubdir, bindSubdir, -1)
 		}
 	}
 

-----------------------------------------------------------------------


hooks/post-receive
--