[ARVADOS] created: 1.1.2-18-g2a3daf1
Git user
git at public.curoverse.com
Mon Jan 8 16:40:27 EST 2018
at 2a3daf14afb93de4d65108019a7a1d35aa1052ad (commit)
commit 2a3daf14afb93de4d65108019a7a1d35aa1052ad
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Jan 8 16:39:15 2018 -0500
12913: Apply secondaryFiles to inputs to RunInSingleContainer
Fixes bug where mounts mismatch cwl.input.yml due to discovering
secondaryFiles during workflow execution.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 316af0e..d426c1a 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -17,7 +17,8 @@ from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
import ruamel.yaml as yaml
-from .runner import upload_dependencies, packed_workflow, upload_workflow_collection, trim_anonymous_location, remove_redundant_fields
+from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
+ trim_anonymous_location, remove_redundant_fields, discover_secondary_files)
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool
from .perf import Perf
@@ -88,6 +89,8 @@ class ArvadosWorkflow(Workflow):
raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
document_loader, workflowobj, uri = (self.doc_loader, self.doc_loader.fetch(self.tool["id"]), self.tool["id"])
+ discover_secondary_files(self.tool["inputs"], joborder)
+
with Perf(metrics, "subworkflow upload_deps"):
upload_dependencies(self.arvrunner,
os.path.basename(joborder.get("id", "#")),
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index c55e976..28de7f3 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -191,12 +191,8 @@ def tag_git_version(packed):
packed["http://schema.org/version"] = githash
-def upload_job_order(arvrunner, name, tool, job_order):
- """Upload local files referenced in the input object and return updated input
- object with 'location' updated to the proper keep references.
- """
-
- for t in tool.tool["inputs"]:
+def discover_secondary_files(inputs, job_order):
+ for t in inputs:
def setSecondary(fileobj):
if isinstance(fileobj, dict) and fileobj.get("class") == "File":
if "secondaryFiles" not in fileobj:
@@ -209,6 +205,13 @@ def upload_job_order(arvrunner, name, tool, job_order):
if shortname(t["id"]) in job_order and t.get("secondaryFiles"):
setSecondary(job_order[shortname(t["id"])])
+def upload_job_order(arvrunner, name, tool, job_order):
+ """Upload local files referenced in the input object and return updated input
+ object with 'location' updated to the proper keep references.
+ """
+
+ discover_secondary_files(tool.tool["inputs"], job_order)
+
jobmapper = upload_dependencies(arvrunner,
name,
tool.doc_loader,
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list