[ARVADOS] updated: 1.1.2-49-gaa04aa1
Git user
git at public.curoverse.com
Fri Jan 12 11:43:09 EST 2018
Summary of changes:
services/crunch-run/crunchrun.go | 41 ++++++++++++++++++++++++++--------------
1 file changed, 27 insertions(+), 14 deletions(-)
via aa04aa117da273ac363453d2297e3474407ac573 (commit)
from 790688e4bcd6f1a92ac409c0c354acba752aaa9c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit aa04aa117da273ac363453d2297e3474407ac573
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jan 12 10:53:57 2018 -0500
12934: Add .keep via manifest manipulation instead of creating real files
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 59fdd00..a6c71fb 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -1050,6 +1050,10 @@ func (runner *ContainerRunner) UploadOutputFile(
relocateTo string,
followed int) (manifestText string, err error) {
+ if infoerr != nil {
+ return "", infoerr
+ }
+
if info.Mode().IsDir() {
// if empty, need to create a .keep file
dir, direrr := os.Open(path)
@@ -1058,21 +1062,23 @@ func (runner *ContainerRunner) UploadOutputFile(
}
defer dir.Close()
names, eof := dir.Readdirnames(1)
- if len(names) == 0 && eof == io.EOF {
- keep, keeperr := os.Create(path+"/.keep")
- if keeperr != nil {
- return "", keeperr
+ if len(names) == 0 && eof == io.EOF && path != runner.HostOutputDir {
+ containerPath := runner.OutputPath + path[len(runner.HostOutputDir):]
+ for _, bind := range binds {
+ mnt := runner.Container.Mounts[bind]
+ // Check if there is a bind for this
+ // directory, in which case assume we don't need .keep
+ runner.CrunchLog.Printf("%v %v", path, bind)
+ if (containerPath == bind || strings.HasPrefix(containerPath, bind+"/")) && mnt.PortableDataHash != "d41d8cd98f00b204e9800998ecf8427e+0" {
+ return
+ }
}
- keep.Close()
+ outputSuffix := path[len(runner.HostOutputDir)+1:]
+ return fmt.Sprintf("./%v d41d8cd98f00b204e9800998ecf8427e+0 0:0:.keep\n", outputSuffix), nil
}
-
return
}
- if infoerr != nil {
- return "", infoerr
- }
-
if followed >= limitFollowSymlinks {
// Got stuck in a loop or just a pathological number of
// directory links, give up.
@@ -1080,9 +1086,16 @@ func (runner *ContainerRunner) UploadOutputFile(
return
}
- // When following symlinks, the source path may need to be logically
- // relocated to some other path within the output collection. Remove
- // the relocateFrom prefix and replace it with relocateTo.
+ // "path" is the actual path we are visiting
+ // "tgt" is the target of "path" (a non-symlink) after following symlinks
+ // "relocated" is the path in the output manifest where the file should be placed,
+ // but has HostOutputDir as a prefix.
+
+ // The destination path in the output manifest may need to be
+ // logically relocated to some other path in order to appear
+ // in the correct location as a result of following a symlink.
+ // Remove the relocateFrom prefix and replace it with
+ // relocateTo.
relocated := relocateTo + path[len(relocateFrom):]
tgt, readlinktgt, info, derefErr := runner.derefOutputSymlink(path, info)
@@ -1103,7 +1116,7 @@ func (runner *ContainerRunner) UploadOutputFile(
// Terminates in this keep mount, so add the
// manifest text at appropriate location.
- outputSuffix := path[len(runner.HostOutputDir):]
+ outputSuffix := relocated[len(runner.HostOutputDir):]
manifestText, err = runner.getCollectionManifestForPath(adjustedMount, outputSuffix)
return
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list