[ARVADOS] updated: 1.1.2-169-gd0d191b

[Git user]

Summary of changes:
 sdk/cwl/arvados_cwl/arvworkflow.py | 64 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 62 insertions(+), 2 deletions(-)

       via  d0d191b2922e4fc5a202c5438c7191a4cc2f6ccc (commit)
      from  54e5ad11af9ee6804c908e49249edf87de7b35dd (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit d0d191b2922e4fc5a202c5438c7191a4cc2f6ccc
Author: jiayong2 <jiayong at math.mit.edu>
Date:   Mon Feb 12 20:03:56 2018 +0000

    Add changes to arvworkflow.py for taking the max requirement
    
    Arvados-DCO-1.1-Signed-off-by: Jiayong Li <jiayong at math.mit.edu>

diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index f0f9c77..15704db 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -14,6 +14,7 @@ from cwltool.load_tool import fetch_document
 from cwltool.process import shortname
 from cwltool.workflow import Workflow, WorkflowException
 from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, visit_class
+from cwltool.builder import Builder
 
 import ruamel.yaml as yaml
 
@@ -71,6 +72,32 @@ def dedup_reqs(reqs):
             dedup[r["class"]] = r
     return [dedup[r] for r in sorted(dedup.keys())]
 
+def get_max_res_req(res_reqs):
+    """Take the max of a list of ResourceRequirement."""
+
+    total_res_req = {}
+    exception_msgs = []
+    for a in ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdirMax", "outdirMin", "outdirMax"):
+        total_res_req[a] = []
+        for res_req in res_reqs:
+            if a in res_req:
+                if isinstance(res_req[a], int): # integer check
+                    total_res_req[a].append(res_req[a])
+                else:
+                    msg = SourceLine(res_req).makeError(
+                    "Non-top-level ResourceRequirement in single container cannot have expressions")
+                    exception_msgs.append(msg)
+    if exception_msgs:
+        raise WorkflowException("\n".join(exception_msgs))
+    else:
+        max_res_req = {}
+        for a in total_res_req:
+            if total_res_req[a]:
+                max_res_req[a] = max(total_res_req[a])
+        if max_res_req:
+            max_res_req["class"] = "ResourceRequirement"
+        return cmap(max_res_req)
+
 class ArvadosWorkflow(Workflow):
     """Wrap cwltool Workflow to override selected methods."""
 
@@ -105,6 +132,39 @@ class ArvadosWorkflow(Workflow):
 
                     packed = pack(document_loader, workflowobj, uri, self.metadata)
 
+                    builder = Builder()
+                    builder.job = joborder
+                    builder.requirements = self.requirements
+                    builder.hints = self.hints
+                    builder.resources = {}
+
+                    res_reqs = {"requirements": [], "hints": []}
+                    for t in ("requirements", "hints"):
+                        for item in packed["$graph"]:
+                            if t in item:
+                                if item["id"] == "#main": # evaluate potential expressions in the top-level requirements/hints
+                                    for req in item[t]:
+                                        if req["class"] == "ResourceRequirement":
+                                            eval_req = {"class": "ResourceRequirement"}
+                                            for a in ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdirMax", "outdirMin", "outdirMax"):
+                                                if a in req:
+                                                    eval_req[a] = builder.do_eval(req[a])
+                                            res_reqs[t].append(eval_req)
+                                else:
+                                    for req in item[t]:
+                                        if req["class"] == "ResourceRequirement":
+                                            res_reqs[t].append(req)
+                    max_res_req = {"requirements": get_max_res_req(res_reqs["requirements"]),
+                                   "hints": get_max_res_req(res_reqs["hints"])}
+
+                    new_spec = {"requirements": self.requirements, "hints": self.hints}
+                    for t in ("requirements", "hints"):
+                        for req in new_spec[t]:
+                            if req["class"] == "ResourceRequirement":
+                                new_spec[t].remove(req)
+                        if max_res_req[t]:
+                            new_spec[t].append(max_res_req[t])
+
                     upload_dependencies(self.arvrunner,
                                         kwargs.get("name", ""),
                                         document_loader,
@@ -158,7 +218,7 @@ class ArvadosWorkflow(Workflow):
                 "inputs": self.tool["inputs"],
                 "outputs": self.tool["outputs"],
                 "stdout": "cwl.output.json",
-                "requirements": self.requirements+[
+                "requirements": new_spec["requirements"]+[
                     {
                     "class": "InitialWorkDirRequirement",
                     "listing": [{
@@ -172,7 +232,7 @@ class ArvadosWorkflow(Workflow):
                             "entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
                         }]
                 }],
-                "hints": self.hints,
+                "hints": new_spec["hints"],
                 "arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
                 "id": "#"
             })

-----------------------------------------------------------------------


hooks/post-receive
--