[ARVADOS] created: 5c274cd1249ce0f5e0048f8e0974e36deee8c9cc
Git user
git at public.curoverse.com
Tue Nov 8 14:06:14 EST 2016
at 5c274cd1249ce0f5e0048f8e0974e36deee8c9cc (commit)
commit 5c274cd1249ce0f5e0048f8e0974e36deee8c9cc
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 8 14:06:08 2016 -0500
10460: Add implied secondaryFiles based on input parameter spec.
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 6f157db..fd150d7 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -9,9 +9,11 @@ from cStringIO import StringIO
import cwltool.draft2tool
from cwltool.draft2tool import CommandLineTool
import cwltool.workflow
-from cwltool.process import get_feature, scandeps, UnsupportedRequirement, normalizeFilesDirs
+from cwltool.process import get_feature, scandeps, UnsupportedRequirement, normalizeFilesDirs, shortname
from cwltool.load_tool import fetch_document
from cwltool.pathmapper import adjustFileObjs, adjustDirObjs
+from cwltool.utils import aslist
+from cwltool.builder import substitute
import arvados.collection
import ruamel.yaml as yaml
@@ -116,6 +118,17 @@ def upload_docker(arvrunner, tool):
def upload_instance(arvrunner, name, tool, job_order):
upload_docker(arvrunner, tool)
+ for t in tool.tool["inputs"]:
+ def setSecondary(fileobj):
+ if "__norecurse" in fileobj:
+ del fileobj["__norecurse"]
+ return
+ if "secondaryFiles" not in fileobj:
+ fileobj["secondaryFiles"] = [{"location": substitute(fileobj["location"], sf), "class": "File", "__norecurse": True} for sf in t["secondaryFiles"]]
+
+ if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
+ adjustFileObjs(job_order, setSecondary)
+
workflowmapper = upload_dependencies(arvrunner,
name,
tool.doc_loader,
commit 97eb0ce9a8a83af1acff17d3dfb66c6ca7522678
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 8 13:42:16 2016 -0500
10466: Report missing 'location' in File object as workflow error.
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 8eb8fe6..ce633d4 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -87,6 +87,8 @@ class ArvadosWorkflow(Workflow):
joborder_keepmount = copy.deepcopy(joborder)
def keepmount(obj):
+ if "location" not in obj:
+ raise WorkflowException("%s object is missing required 'location' field: %s" % (obj["class"], obj))
if obj["location"].startswith("keep:"):
obj["location"] = "/keep/" + obj["location"][5:]
if "listing" in obj:
commit b03f167f42f9803814e2d22c4de68a30a3bc9dae
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Nov 8 13:00:36 2016 -0500
10460: Fix initialWorkDir file staging broken by change in visit() behavior for directories.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 5ae2de3..92be92d 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -232,7 +232,7 @@ class ArvCwlRunner(object):
def rewrite(fileobj):
fileobj["location"] = generatemapper.mapper(fileobj["location"]).target
- for k in ("basename", "size", "listing", "contents"):
+ for k in ("basename", "listing", "contents"):
if k in fileobj:
del fileobj[k]
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index c15b289..58500d3 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -150,27 +150,31 @@ class ArvPathMapper(PathMapper):
else:
return super(ArvPathMapper, self).reversemap(target)
-class InitialWorkDirPathMapper(PathMapper):
+class StagingPathMapper(PathMapper):
+ _follow_dirs = True
def visit(self, obj, stagedir, basedir, copy=False):
# type: (Dict[unicode, Any], unicode, unicode, bool) -> None
loc = obj["location"]
+ tgt = os.path.join(stagedir, obj["basename"])
if obj["class"] == "Directory":
- self._pathmap[loc] = MapperEnt(obj["location"], stagedir, "Directory")
- self.visitlisting(obj.get("listing", []), stagedir, basedir)
+ self._pathmap[loc] = MapperEnt(loc, tgt, "Directory")
+ if loc.startswith("_:") or self._follow_dirs:
+ self.visitlisting(obj.get("listing", []), tgt, basedir)
elif obj["class"] == "File":
if loc in self._pathmap:
return
- tgt = os.path.join(stagedir, obj["basename"])
- if "contents" in obj and obj["location"].startswith("_:"):
+ if "contents" in obj and loc.startswith("_:"):
self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile")
else:
if copy:
- self._pathmap[loc] = MapperEnt(obj["path"], tgt, "WritableFile")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "WritableFile")
else:
- self._pathmap[loc] = MapperEnt(obj["path"], tgt, "File")
+ self._pathmap[loc] = MapperEnt(loc, tgt, "File")
self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir)
+
+class InitialWorkDirPathMapper(StagingPathMapper):
def setup(self, referenced_files, basedir):
# type: (List[Any], unicode) -> None
@@ -183,24 +187,8 @@ class InitialWorkDirPathMapper(PathMapper):
self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type)
-class FinalOutputPathMapper(PathMapper):
- def visit(self, obj, stagedir, basedir, copy=False):
- # type: (Dict[unicode, Any], unicode, unicode, bool) -> None
- loc = obj["location"]
- tgt = os.path.join(stagedir, obj["basename"])
- if obj["class"] == "Directory":
- self._pathmap[loc] = MapperEnt(tgt, tgt, "Directory")
- if loc.startswith("_:"):
- self.visitlisting(obj.get("listing", []), tgt, basedir)
- elif obj["class"] == "File":
- if loc in self._pathmap:
- return
- if "contents" in obj and loc.startswith("_:"):
- self._pathmap[loc] = MapperEnt(obj["contents"], tgt, "CreateFile")
- else:
- self._pathmap[loc] = MapperEnt(loc, tgt, "File")
- self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir)
-
+class FinalOutputPathMapper(StagingPathMapper):
+ _follow_dirs = False
def setup(self, referenced_files, basedir):
# type: (List[Any], unicode) -> None
self.visitlisting(referenced_files, self.stagedir, basedir)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list