[ARVADOS] created: 1.1.2-55-ge761c21
Git user
git at public.curoverse.com
Fri Jan 12 14:13:39 EST 2018
at e761c211063177d544c43e79abc8d184709ff79f (commit)
commit e761c211063177d544c43e79abc8d184709ff79f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jan 12 14:12:01 2018 -0500
12764: Update documentation.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/doc/_includes/_mount_types.liquid b/doc/_includes/_mount_types.liquid
index 734b07c..c4e751a 100644
--- a/doc/_includes/_mount_types.liquid
+++ b/doc/_includes/_mount_types.liquid
@@ -64,7 +64,7 @@ When a container's output_path is a tmp mount backed by local disk, this output
1. Only mount points of kind @collection@ are supported.
-2. Mount points underneath output_path must not use @"writable":true at . If any of them are set as @writable@, the API will refuse to create/update the container request, and crunch-run will fail the container.
+2. Mount points underneath output_path which have "writable":true will be copied into output_path the mount point during container initialization. These files may be updated, renamed, or deleted by the running container. On container completetion, the (possibly modified) copies are saved to the output collection.
3. If any such mount points are configured as @exclude_from_output":true@, they will be excluded from the output.
commit 03e6181a6e6e0bd0c8e5a380f509963b76bb1fd3
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jan 12 14:06:06 2018 -0500
12764: Support empty anonymous directories by creating a .keep placeholder.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index eb04dda..871d4c3 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -226,13 +226,12 @@ class ArvCwlRunner(object):
def check_features(self, obj):
if isinstance(obj, dict):
- #if obj.get("writable"):
- # raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
+ if obj.get("writable") and self.work_api != "containers":
+ raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
if obj.get("class") == "DockerRequirement":
- if obj.get("dockerOutputDirectory"):
- # TODO: can be supported by containers API, but not jobs API.
+ if obj.get("dockerOutputDirectory") and self.work_api != "containers":
raise SourceLine(obj, "dockerOutputDirectory", UnsupportedRequirement).makeError(
- "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
+ "Option 'dockerOutputDirectory' of DockerRequirement not supported with --api=jobs.")
for v in obj.itervalues():
self.check_features(v)
elif isinstance(obj, list):
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index fa47666..f7959fb 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -105,17 +105,32 @@ class ArvadosContainer(object):
generatemapper = NoFollowPathMapper([self.generatefiles], "", "",
separateDirs=False)
+ logger.debug("generatemapper is %s", generatemapper._pathmap)
+
with Perf(metrics, "createfiles %s" % self.name):
for f, p in generatemapper.items():
if not p.target:
pass
elif p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
- source, path = self.arvrunner.fs_access.get_collection(p.resolved)
- vwd.copy(path, p.target, source_collection=source)
+ if p.resolved.startswith("_:"):
+ vwd.mkdirs(p.target)
+ else:
+ source, path = self.arvrunner.fs_access.get_collection(p.resolved)
+ vwd.copy(path, p.target, source_collection=source)
elif p.type == "CreateFile":
with vwd.open(p.target, "w") as n:
n.write(p.resolved.encode("utf-8"))
+ def keepemptydirs(p):
+ if isinstance(p, arvados.collection.RichCollectionBase):
+ if len(p) == 0:
+ p.open(".keep", "w").close()
+ else:
+ for c in p:
+ keepemptydirs(p[c])
+
+ keepemptydirs(vwd)
+
with Perf(metrics, "generatefiles.save_new %s" % self.name):
vwd.save_new()
commit bba90dbdb5b5b7cab5c4e83b141d4477f0174884
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 10 17:43:49 2018 -0500
12764: Pass one cwl test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 5a8382f..0aec847 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -396,7 +396,7 @@ func copyfile(src string, dst string) (err error) {
if err != nil {
return
}
- _, err = io.Copy(srcfile, dstfile)
+ _, err = io.Copy(dstfile, srcfile)
if err != nil {
return
}
@@ -1163,7 +1163,7 @@ func (runner *ContainerRunner) UploadOutputFile(
// go through mounts and try reverse map to collection reference
for _, bind := range binds {
mnt := runner.Container.Mounts[bind]
- if tgt == bind || strings.HasPrefix(tgt, bind+"/") {
+ if (tgt == bind || strings.HasPrefix(tgt, bind+"/")) && !mnt.Writable {
// get path relative to bind
targetSuffix := tgt[len(bind):]
@@ -1306,7 +1306,7 @@ func (runner *ContainerRunner) CaptureOutput() error {
continue
}
- if mnt.ExcludeFromOutput == true {
+ if mnt.ExcludeFromOutput == true || mnt.Writable {
continue
}
commit 3f262b0de86e33e9d156c20e05c9c63c47e4f323
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 10 16:58:29 2018 -0500
12764: Fixes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 21fd279..fa47666 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -126,7 +126,7 @@ class ArvadosContainer(object):
mounts[mountpoint] = {"kind": "collection",
"portable_data_hash": vwd.portable_data_hash(),
"path": p.target}
- if t.type.startswith("Writable"):
+ if p.type.startswith("Writable"):
mounts[mountpoint]["writable"] = True
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 914ccaa..bb95ba9 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -230,7 +230,10 @@ class StagingPathMapper(PathMapper):
tgt = "%s_%i%s" % (basetgt, n, baseext)
self.targets.add(tgt)
if obj["class"] == "Directory":
- self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
+ if obj.get("writable"):
+ self._pathmap[loc] = MapperEnt(loc, tgt, "WritableDirectory", staged)
+ else:
+ self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
if loc.startswith("_:") or self._follow_dirs:
self.visitlisting(obj.get("listing", []), tgt, basedir)
elif obj["class"] == "File":
@@ -239,7 +242,7 @@ class StagingPathMapper(PathMapper):
if "contents" in obj and loc.startswith("_:"):
self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile", staged)
else:
- if copy:
+ if copy or obj.get("writable"):
self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile", staged)
else:
self._pathmap[loc] = MapperEnt(loc, tgt, "File", staged)
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index a2750f4..5a8382f 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -634,7 +634,10 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if err == nil {
if dir.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
- return copyfile(walkpath, path.Join(cp.bind, walkpath))
+ if walkinfo.Mode().IsRegular() {
+ return copyfile(walkpath, path.Join(cp.bind, walkpath[len(cp.src):]))
+ }
+ return nil
})
} else {
err = copyfile(cp.src, cp.bind)
commit 2fcf85f500c08983aa10aaeaa28f77add5591ef4
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 10 15:44:50 2018 -0500
12764: arvados-cwl-runner support
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index bb01796..eb04dda 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -226,8 +226,8 @@ class ArvCwlRunner(object):
def check_features(self, obj):
if isinstance(obj, dict):
- if obj.get("writable"):
- raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
+ #if obj.get("writable"):
+ # raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
if obj.get("class") == "DockerRequirement":
if obj.get("dockerOutputDirectory"):
# TODO: can be supported by containers API, but not jobs API.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index e8e2a51..21fd279 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -109,7 +109,7 @@ class ArvadosContainer(object):
for f, p in generatemapper.items():
if not p.target:
pass
- elif p.type in ("File", "Directory"):
+ elif p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
source, path = self.arvrunner.fs_access.get_collection(p.resolved)
vwd.copy(path, p.target, source_collection=source)
elif p.type == "CreateFile":
@@ -126,6 +126,8 @@ class ArvadosContainer(object):
mounts[mountpoint] = {"kind": "collection",
"portable_data_hash": vwd.portable_data_hash(),
"path": p.target}
+ if t.type.startswith("Writable"):
+ mounts[mountpoint]["writable"] = True
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
if self.environment:
commit 8266e649dbe0b71554c5208baff50212742c682c
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Dec 7 09:51:25 2017 -0500
12764: Writable file/directory support wip
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index a6c71fb..a2750f4 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -384,6 +384,37 @@ func (runner *ContainerRunner) SetupArvMountPoint(prefix string) (err error) {
return
}
+func copyfile(src string, dst string) (err error) {
+ srcfile, err := os.Open(src)
+ if err != nil {
+ return
+ }
+
+ os.MkdirAll(path.Dir(dst), 0770)
+
+ dstfile, err := os.Create(dst)
+ if err != nil {
+ return
+ }
+ _, err = io.Copy(srcfile, dstfile)
+ if err != nil {
+ return
+ }
+
+ err = srcfile.Close()
+ err2 := dstfile.Close()
+
+ if err != nil {
+ return
+ }
+
+ if err2 != nil {
+ return err2
+ }
+
+ return nil
+}
+
func (runner *ContainerRunner) SetupMounts() (err error) {
err = runner.SetupArvMountPoint("keep")
if err != nil {
@@ -411,6 +442,11 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
runner.Binds = nil
runner.Volumes = make(map[string]struct{})
needCertMount := true
+ type copyFile struct {
+ src string
+ bind string
+ }
+ var copyFiles []copyFile
var binds []string
for bind := range runner.Container.Mounts {
@@ -466,7 +502,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
- if mnt.Writable {
+ if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
return fmt.Errorf("Can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
@@ -493,10 +529,12 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if mnt.Writable {
if bind == runner.Container.OutputPath {
runner.HostOutputDir = src
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
} else if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("Writable mount points are not permitted underneath the output_path: %v", bind)
+ copyFiles = append(copyFiles, copyFile{src, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
+ } else {
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
}
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
} else {
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
}
@@ -591,6 +629,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
}
+ for _, cp := range copyFiles {
+ dir, err := os.Stat(cp.src)
+ if err == nil {
+ if dir.IsDir() {
+ err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
+ return copyfile(walkpath, path.Join(cp.bind, walkpath))
+ })
+ } else {
+ err = copyfile(cp.src, cp.bind)
+ }
+ }
+ if err != nil {
+ return fmt.Errorf("While staging writable files: %v", err)
+ }
+ }
+
return nil
}
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index ab7417e..5f586f8 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1210,20 +1210,25 @@ func (s *TestSuite) TestSetupMounts(c *C) {
checkEmpty()
}
- // Writable mount points are not allowed underneath output_dir mount point
+ // Writable mount points copied to output_dir mount point
{
i = 0
cr.ArvMountPoint = ""
cr.Container.Mounts = make(map[string]arvados.Mount)
cr.Container.Mounts = map[string]arvados.Mount{
- "/tmp": {Kind: "tmp"},
- "/tmp/foo": {Kind: "collection", Writable: true},
+ "/tmp": {Kind: "tmp"},
+ "/tmp/foo": {Kind: "collection",
+ PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
+ Writable: true},
}
cr.OutputPath = "/tmp"
+ os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
+
err := cr.SetupMounts()
- c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
+ c.Check(err, IsNil)
+ _, err = os.Stat(cr.HostOutputDir + "/foo")
+ c.Check(err, IsNil)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list