[ARVADOS] created: 1.1.2-136-g4da7a9b
Git user
git at public.curoverse.com
Mon Jan 29 14:50:33 EST 2018
at 4da7a9b5c11caf085c14cbf0eacdc16b6f6d7fa5 (commit)
commit 4da7a9b5c11caf085c14cbf0eacdc16b6f6d7fa5
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jan 12 15:31:09 2018 -0500
12764: Support 'dockerOutputDirectory' for containers to get to 100% conformance.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index b667dac..52f7d8a 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -36,8 +36,13 @@ class ArvadosCommandTool(CommandLineTool):
def job(self, joborder, output_callback, **kwargs):
if self.work_api == "containers":
- kwargs["outdir"] = "/var/spool/cwl"
- kwargs["docker_outdir"] = "/var/spool/cwl"
+ dockerReq, is_req = self.get_requirement("DockerRequirement")
+ if dockerReq.get("dockerOutputDirectory") and dockerReq.get("dockerOutputDirectory").startswith('/'):
+ kwargs["outdir"] = dockerReq.get("dockerOutputDirectory")
+ kwargs["docker_outdir"] = dockerReq.get("dockerOutputDirectory")
+ else:
+ kwargs["outdir"] = "/var/spool/cwl"
+ kwargs["docker_outdir"] = "/var/spool/cwl"
elif self.work_api == "jobs":
kwargs["outdir"] = "$(task.outdir)"
kwargs["docker_outdir"] = "$(task.outdir)"
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 2ca63cf..fb5d036 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -161,7 +161,7 @@ def upload_docker(arvrunner, tool):
if isinstance(tool, CommandLineTool):
(docker_req, docker_is_req) = get_feature(tool, "DockerRequirement")
if docker_req:
- if docker_req.get("dockerOutputDirectory"):
+ if docker_req.get("dockerOutputDirectory") and arvrunner.work_api != "containers":
# TODO: can be supported by containers API, but not jobs API.
raise SourceLine(docker_req, "dockerOutputDirectory", UnsupportedRequirement).makeError(
"Option 'dockerOutputDirectory' of DockerRequirement not supported.")
commit 4e78fdb8c41ee73a0f14a349fe4edcfdb1523458
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jan 12 14:12:01 2018 -0500
12764: Update documentation.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/doc/_includes/_mount_types.liquid b/doc/_includes/_mount_types.liquid
index 734b07c..c4e751a 100644
--- a/doc/_includes/_mount_types.liquid
+++ b/doc/_includes/_mount_types.liquid
@@ -64,7 +64,7 @@ When a container's output_path is a tmp mount backed by local disk, this output
1. Only mount points of kind @collection@ are supported.
-2. Mount points underneath output_path must not use @"writable":true at . If any of them are set as @writable@, the API will refuse to create/update the container request, and crunch-run will fail the container.
+2. Mount points underneath output_path which have "writable":true will be copied into output_path the mount point during container initialization. These files may be updated, renamed, or deleted by the running container. On container completetion, the (possibly modified) copies are saved to the output collection.
3. If any such mount points are configured as @exclude_from_output":true@, they will be excluded from the output.
commit bf59702692c89a809f102d8bd4b9caf531f4c9be
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Fri Jan 12 14:06:06 2018 -0500
12764: Support empty anonymous directories by creating a .keep placeholder.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 892cb0a..af6ab33 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -224,13 +224,12 @@ class ArvCwlRunner(object):
def check_features(self, obj):
if isinstance(obj, dict):
- #if obj.get("writable"):
- # raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
+ if obj.get("writable") and self.work_api != "containers":
+ raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported with --api=jobs")
if obj.get("class") == "DockerRequirement":
- if obj.get("dockerOutputDirectory"):
- # TODO: can be supported by containers API, but not jobs API.
+ if obj.get("dockerOutputDirectory") and self.work_api != "containers":
raise SourceLine(obj, "dockerOutputDirectory", UnsupportedRequirement).makeError(
- "Option 'dockerOutputDirectory' of DockerRequirement not supported.")
+ "Option 'dockerOutputDirectory' of DockerRequirement not supported with --api=jobs.")
for v in obj.itervalues():
self.check_features(v)
elif isinstance(obj, list):
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index fba158b..abe67c8 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -105,17 +105,32 @@ class ArvadosContainer(object):
generatemapper = NoFollowPathMapper([self.generatefiles], "", "",
separateDirs=False)
+ logger.debug("generatemapper is %s", generatemapper._pathmap)
+
with Perf(metrics, "createfiles %s" % self.name):
for f, p in generatemapper.items():
if not p.target:
pass
elif p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
- source, path = self.arvrunner.fs_access.get_collection(p.resolved)
- vwd.copy(path, p.target, source_collection=source)
+ if p.resolved.startswith("_:"):
+ vwd.mkdirs(p.target)
+ else:
+ source, path = self.arvrunner.fs_access.get_collection(p.resolved)
+ vwd.copy(path, p.target, source_collection=source)
elif p.type == "CreateFile":
with vwd.open(p.target, "w") as n:
n.write(p.resolved.encode("utf-8"))
+ def keepemptydirs(p):
+ if isinstance(p, arvados.collection.RichCollectionBase):
+ if len(p) == 0:
+ p.open(".keep", "w").close()
+ else:
+ for c in p:
+ keepemptydirs(p[c])
+
+ keepemptydirs(vwd)
+
with Perf(metrics, "generatefiles.save_new %s" % self.name):
vwd.save_new()
commit 76a7dd09a977c109de100884500b531e6bbe2166
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 10 17:43:49 2018 -0500
12764: Pass one cwl test
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 5437e36..852d5a2 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -342,7 +342,7 @@ func copyfile(src string, dst string) (err error) {
if err != nil {
return
}
- _, err = io.Copy(srcfile, dstfile)
+ _, err = io.Copy(dstfile, srcfile)
if err != nil {
return
}
@@ -1135,7 +1135,7 @@ func (runner *ContainerRunner) UploadOutputFile(
// go through mounts and try reverse map to collection reference
for _, bind := range binds {
mnt := runner.Container.Mounts[bind]
- if tgt == bind || strings.HasPrefix(tgt, bind+"/") {
+ if (tgt == bind || strings.HasPrefix(tgt, bind+"/")) && !mnt.Writable {
// get path relative to bind
targetSuffix := tgt[len(bind):]
@@ -1274,7 +1274,7 @@ func (runner *ContainerRunner) CaptureOutput() error {
continue
}
- if mnt.ExcludeFromOutput == true {
+ if mnt.ExcludeFromOutput == true || mnt.Writable {
continue
}
commit 11eda09c7340da2c0a01259e3711758782c9b1c9
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 10 16:58:29 2018 -0500
12764: Fixes
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index deccc69..fba158b 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -126,7 +126,7 @@ class ArvadosContainer(object):
mounts[mountpoint] = {"kind": "collection",
"portable_data_hash": vwd.portable_data_hash(),
"path": p.target}
- if t.type.startswith("Writable"):
+ if p.type.startswith("Writable"):
mounts[mountpoint]["writable"] = True
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 914ccaa..bb95ba9 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -230,7 +230,10 @@ class StagingPathMapper(PathMapper):
tgt = "%s_%i%s" % (basetgt, n, baseext)
self.targets.add(tgt)
if obj["class"] == "Directory":
- self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
+ if obj.get("writable"):
+ self._pathmap[loc] = MapperEnt(loc, tgt, "WritableDirectory", staged)
+ else:
+ self._pathmap[loc] = MapperEnt(loc, tgt, "Directory", staged)
if loc.startswith("_:") or self._follow_dirs:
self.visitlisting(obj.get("listing", []), tgt, basedir)
elif obj["class"] == "File":
@@ -239,7 +242,7 @@ class StagingPathMapper(PathMapper):
if "contents" in obj and loc.startswith("_:"):
self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile", staged)
else:
- if copy:
+ if copy or obj.get("writable"):
self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile", staged)
else:
self._pathmap[loc] = MapperEnt(loc, tgt, "File", staged)
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 60f2a6f..5437e36 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -580,7 +580,10 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if err == nil {
if dir.IsDir() {
err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
- return copyfile(walkpath, path.Join(cp.bind, walkpath))
+ if walkinfo.Mode().IsRegular() {
+ return copyfile(walkpath, path.Join(cp.bind, walkpath[len(cp.src):]))
+ }
+ return nil
})
} else {
err = copyfile(cp.src, cp.bind)
commit bd1378553152e98ffd0e9ecf68dc0f92ea3ff186
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Wed Jan 10 15:44:50 2018 -0500
12764: arvados-cwl-runner support
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index e82fd9f..892cb0a 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -224,8 +224,8 @@ class ArvCwlRunner(object):
def check_features(self, obj):
if isinstance(obj, dict):
- if obj.get("writable"):
- raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
+ #if obj.get("writable"):
+ # raise SourceLine(obj, "writable", UnsupportedRequirement).makeError("InitialWorkDir feature 'writable: true' not supported")
if obj.get("class") == "DockerRequirement":
if obj.get("dockerOutputDirectory"):
# TODO: can be supported by containers API, but not jobs API.
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 014e1b9..deccc69 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -109,7 +109,7 @@ class ArvadosContainer(object):
for f, p in generatemapper.items():
if not p.target:
pass
- elif p.type in ("File", "Directory"):
+ elif p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
source, path = self.arvrunner.fs_access.get_collection(p.resolved)
vwd.copy(path, p.target, source_collection=source)
elif p.type == "CreateFile":
@@ -126,6 +126,8 @@ class ArvadosContainer(object):
mounts[mountpoint] = {"kind": "collection",
"portable_data_hash": vwd.portable_data_hash(),
"path": p.target}
+ if t.type.startswith("Writable"):
+ mounts[mountpoint]["writable"] = True
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
if self.environment:
commit 25da01646d0fc0d5b49cdeaa10173beabfd2e8b9
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Dec 7 09:51:25 2017 -0500
12764: Writable file/directory support wip
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 7eefb1a..60f2a6f 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -330,6 +330,37 @@ func (runner *ContainerRunner) SetupArvMountPoint(prefix string) (err error) {
return
}
+func copyfile(src string, dst string) (err error) {
+ srcfile, err := os.Open(src)
+ if err != nil {
+ return
+ }
+
+ os.MkdirAll(path.Dir(dst), 0770)
+
+ dstfile, err := os.Create(dst)
+ if err != nil {
+ return
+ }
+ _, err = io.Copy(srcfile, dstfile)
+ if err != nil {
+ return
+ }
+
+ err = srcfile.Close()
+ err2 := dstfile.Close()
+
+ if err != nil {
+ return
+ }
+
+ if err2 != nil {
+ return err2
+ }
+
+ return nil
+}
+
func (runner *ContainerRunner) SetupMounts() (err error) {
err = runner.SetupArvMountPoint("keep")
if err != nil {
@@ -357,6 +388,11 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
runner.Binds = nil
runner.Volumes = make(map[string]struct{})
needCertMount := true
+ type copyFile struct {
+ src string
+ bind string
+ }
+ var copyFiles []copyFile
var binds []string
for bind := range runner.Container.Mounts {
@@ -412,7 +448,7 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
pdhOnly = false
src = fmt.Sprintf("%s/by_id/%s", runner.ArvMountPoint, mnt.UUID)
} else if mnt.PortableDataHash != "" {
- if mnt.Writable {
+ if mnt.Writable && !strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
return fmt.Errorf("Can never write to a collection specified by portable data hash")
}
idx := strings.Index(mnt.PortableDataHash, "/")
@@ -439,10 +475,12 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
if mnt.Writable {
if bind == runner.Container.OutputPath {
runner.HostOutputDir = src
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
} else if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
- return fmt.Errorf("Writable mount points are not permitted underneath the output_path: %v", bind)
+ copyFiles = append(copyFiles, copyFile{src, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
+ } else {
+ runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
}
- runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s", src, bind))
} else {
runner.Binds = append(runner.Binds, fmt.Sprintf("%s:%s:ro", src, bind))
}
@@ -537,6 +575,22 @@ func (runner *ContainerRunner) SetupMounts() (err error) {
}
}
+ for _, cp := range copyFiles {
+ dir, err := os.Stat(cp.src)
+ if err == nil {
+ if dir.IsDir() {
+ err = filepath.Walk(cp.src, func(walkpath string, walkinfo os.FileInfo, walkerr error) error {
+ return copyfile(walkpath, path.Join(cp.bind, walkpath))
+ })
+ } else {
+ err = copyfile(cp.src, cp.bind)
+ }
+ }
+ if err != nil {
+ return fmt.Errorf("While staging writable files: %v", err)
+ }
+ }
+
return nil
}
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index 22989bb..5965a7d 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -1233,20 +1233,25 @@ func (s *TestSuite) TestSetupMounts(c *C) {
checkEmpty()
}
- // Writable mount points are not allowed underneath output_dir mount point
+ // Writable mount points copied to output_dir mount point
{
i = 0
cr.ArvMountPoint = ""
cr.Container.Mounts = make(map[string]arvados.Mount)
cr.Container.Mounts = map[string]arvados.Mount{
- "/tmp": {Kind: "tmp"},
- "/tmp/foo": {Kind: "collection", Writable: true},
+ "/tmp": {Kind: "tmp"},
+ "/tmp/foo": {Kind: "collection",
+ PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
+ Writable: true},
}
cr.OutputPath = "/tmp"
+ os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
+
err := cr.SetupMounts()
- c.Check(err, NotNil)
- c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
+ c.Check(err, IsNil)
+ _, err = os.Stat(cr.HostOutputDir + "/foo")
+ c.Check(err, IsNil)
os.RemoveAll(cr.ArvMountPoint)
cr.CleanupDirs()
checkEmpty()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list