[ARVADOS] updated: 1.3.0-2774-g339da6b41
Git user
git at public.arvados.org
Mon Jul 20 22:11:34 UTC 2020
Summary of changes:
sdk/cwl/arvados_cwl/arvcontainer.py | 28 ++++++++++++++++++++--------
sdk/cwl/arvados_cwl/pathmapper.py | 1 +
sdk/cwl/setup.py | 2 +-
3 files changed, 22 insertions(+), 9 deletions(-)
via 339da6b41fe014db93bd123cc9285cbeefe16936 (commit)
via 26a86a0239e5d626b1e02163aa417678c58e97ac (commit)
from 150a601dbcaf6b999e54bd17d5152f15b626c7f3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 339da6b41fe014db93bd123cc9285cbeefe16936
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Jul 20 18:11:18 2020 -0400
16353: Handle a few mount point edge cases.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 376116416..fb23c2ccf 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -152,10 +152,10 @@ class ArvadosContainer(JobBase):
if not p.target:
continue
- if not p.target.startswith("/"):
- raise Exception("Expected mount target to start with '/'")
-
- dst = p.target[len(self.outdir)+1:] if p.target.startswith(self.outdir+"/") else p.target[1:]
+ if p.target.startswith("/"):
+ dst = p.target[len(self.outdir)+1:] if p.target.startswith(self.outdir+"/") else p.target[1:]
+ else:
+ dst = p.target
if p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
if p.resolved.startswith("_:"):
@@ -165,7 +165,8 @@ class ArvadosContainer(JobBase):
vwd.copy(path or ".", dst, source_collection=source)
elif p.type == "CreateFile":
if self.arvrunner.secret_store.has_secret(p.resolved):
- secret_mounts[p.target] = {
+ mountpoint = p.target if p.target.startswith("/") else os.path.join(self.outdir, p.target)
+ secret_mounts[mountpoint] = {
"kind": "text",
"content": self.arvrunner.secret_store.retrieve(p.resolved)
}
@@ -197,12 +198,16 @@ class ArvadosContainer(JobBase):
if (not p.target or self.arvrunner.secret_store.has_secret(p.resolved) or
(prev is not None and p.target.startswith(prev))):
continue
- dst = p.target[len(self.outdir)+1:] if p.target.startswith(self.outdir+"/") else p.target[1:]
- mounts[p.target] = {"kind": "collection",
+ if p.target.startswith("/"):
+ dst = p.target[len(self.outdir)+1:] if p.target.startswith(self.outdir+"/") else p.target[1:]
+ else:
+ dst = p.target
+ mountpoint = p.target if p.target.startswith("/") else os.path.join(self.outdir, p.target)
+ mounts[mountpoint] = {"kind": "collection",
"portable_data_hash": vwd.portable_data_hash(),
"path": dst}
if p.type.startswith("Writable"):
- mounts[p.target]["writable"] = True
+ mounts[mountpoint]["writable"] = True
prev = p.target + "/"
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
commit 26a86a0239e5d626b1e02163aa417678c58e97ac
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Jul 20 15:59:29 2020 -0400
16353: Support CWL 1.2 behavior for mounting at arbitrary container path
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 2b55ce9df..376116416 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -150,21 +150,27 @@ class ArvadosContainer(JobBase):
with Perf(metrics, "createfiles %s" % self.name):
for f, p in sorteditems:
if not p.target:
- pass
- elif p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
+ continue
+
+ if not p.target.startswith("/"):
+ raise Exception("Expected mount target to start with '/'")
+
+ dst = p.target[len(self.outdir)+1:] if p.target.startswith(self.outdir+"/") else p.target[1:]
+
+ if p.type in ("File", "Directory", "WritableFile", "WritableDirectory"):
if p.resolved.startswith("_:"):
- vwd.mkdirs(p.target)
+ vwd.mkdirs(dst)
else:
source, path = self.arvrunner.fs_access.get_collection(p.resolved)
- vwd.copy(path or ".", p.target, source_collection=source)
+ vwd.copy(path or ".", dst, source_collection=source)
elif p.type == "CreateFile":
if self.arvrunner.secret_store.has_secret(p.resolved):
- secret_mounts["%s/%s" % (self.outdir, p.target)] = {
+ secret_mounts[p.target] = {
"kind": "text",
"content": self.arvrunner.secret_store.retrieve(p.resolved)
}
else:
- with vwd.open(p.target, "w") as n:
+ with vwd.open(dst, "w") as n:
n.write(p.resolved)
def keepemptydirs(p):
@@ -191,12 +197,12 @@ class ArvadosContainer(JobBase):
if (not p.target or self.arvrunner.secret_store.has_secret(p.resolved) or
(prev is not None and p.target.startswith(prev))):
continue
- mountpoint = "%s/%s" % (self.outdir, p.target)
- mounts[mountpoint] = {"kind": "collection",
+ dst = p.target[len(self.outdir)+1:] if p.target.startswith(self.outdir+"/") else p.target[1:]
+ mounts[p.target] = {"kind": "collection",
"portable_data_hash": vwd.portable_data_hash(),
- "path": p.target}
+ "path": dst}
if p.type.startswith("Writable"):
- mounts[mountpoint]["writable"] = True
+ mounts[p.target]["writable"] = True
prev = p.target + "/"
container_request["environment"] = {"TMPDIR": self.tmpdir, "HOME": self.outdir}
@@ -316,6 +322,7 @@ class ArvadosContainer(JobBase):
logger.info("%s %s state is %s", self.arvrunner.label(self), response["uuid"], response["state"])
except Exception:
logger.exception("%s got an error", self.arvrunner.label(self))
+ logger.debug("Container request was %s", container_request)
self.output_callback({}, "permanentFail")
def done(self, record):
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 4cd204f7d..5bad29077 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -285,6 +285,7 @@ class StagingPathMapper(PathMapper):
def visit(self, obj, stagedir, basedir, copy=False, staged=False):
# type: (Dict[unicode, Any], unicode, unicode, bool) -> None
loc = obj["location"]
+ stagedir = obj.get("dirname") or stagedir
tgt = os.path.join(stagedir, obj["basename"])
basetgt, baseext = os.path.splitext(tgt)
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index ea63ec485..d703fcbc5 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,7 +39,7 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200710214758',
+ 'cwltool==3.0.20200720165847',
'schema-salad==7.0.20200612160654',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list