[ARVADOS] updated: cb214c45eceb71882e47f619eb5233d83fd949f4

[Git user]

Summary of changes:
 sdk/cwl/arvados_cwl/__init__.py     |  2 +-
 sdk/cwl/arvados_cwl/arvcontainer.py |  2 +-
 sdk/cwl/arvados_cwl/arvjob.py       |  6 ++++--
 sdk/cwl/arvados_cwl/done.py         | 28 +++++++++++++++++++++++-----
 4 files changed, 29 insertions(+), 9 deletions(-)

       via  cb214c45eceb71882e47f619eb5233d83fd949f4 (commit)
       via  1fb7405bd258093fa5425f6626dec42a3ab3e9c7 (commit)
       via  59f984237002e1c49f9667d695a0f5f3b35d034e (commit)
      from  dfc278ebe50c5a8dc9eb98a98539efb2f942e982 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit cb214c45eceb71882e47f619eb5233d83fd949f4
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Jan 16 10:22:30 2017 -0500

    10812: Fix imports, reference #main in cwl:tool

diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index f68d99e..bf1052c 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -4,13 +4,15 @@ import copy
 import json
 import time
 
-from cwltool.process import get_feature, shortname
+from cwltool.process import get_feature, shortname, UnsupportedRequirement
 from cwltool.errors import WorkflowException
 from cwltool.draft2tool import revmap_file, CommandLineTool
 from cwltool.load_tool import fetch_document
 from cwltool.builder import Builder
 from cwltool.pathmapper import adjustDirObjs
 
+from schema_salad.sourceline import SourceLine
+
 import ruamel.yaml as yaml
 
 import arvados.collection
@@ -271,7 +273,7 @@ class RunnerJob(Runner):
         else:
             packed = packed_workflow(self.arvrunner, self.tool)
             wf_pdh = self.upload_workflow_collection(packed)
-            self.job_order["cwl:tool"] = "%s/workflow.cwl" % wf_pdh
+            self.job_order["cwl:tool"] = "%s/workflow.cwl#main" % wf_pdh
 
         adjustDirObjs(self.job_order, trim_listing)
 

commit 1fb7405bd258093fa5425f6626dec42a3ab3e9c7
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jan 12 17:20:05 2017 -0500

    10812: Bugfixes

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 66ac000..a5fa4e0 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -28,7 +28,7 @@ from arvados.errors import ApiError
 
 from .arvcontainer import ArvadosContainer, RunnerContainer
 from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
-from. runner import Runner, upload_docker, upload_job_order, upload_workflow_deps
+from. runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, upload_dependencies
 from .arvtool import ArvadosCommandTool
 from .arvworkflow import ArvadosWorkflow, upload_workflow
 from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 51e5956..6015c1d 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -236,7 +236,7 @@ class RunnerContainer(Runner):
             workflowpath = "/var/lib/cwl/workflow.json#main"
             container_req["mounts"]["/var/lib/cwl/workflow.json"] = {
                 "kind": "json",
-                "json": packed
+                "content": packed
             }
             if self.tool.tool.get("id", "").startswith("arvwf:"):
                 container_req["properties"]["template_uuid"] = self.tool.tool["id"][6:33]

commit 59f984237002e1c49f9667d695a0f5f3b35d034e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jan 12 17:05:23 2017 -0500

    10812: Perform merge of logs based on timestamp instead of sorting log lines.

diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 87908c2..15068b8 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -48,21 +48,39 @@ crunchstat_re = re.compile(r"^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d [a-z0-9]{5}-8i9sb-[
 timestamp_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z) (.*)")
 
 def logtail(logcollection, logger, header, maxlen=25):
-    logtail = deque([], maxlen*len(logcollection))
     containersapi = ("crunch-run.txt" in logcollection)
+    mergelogs = {}
 
     for log in logcollection.keys():
         if not containersapi or log in ("crunch-run.txt", "stdout.txt", "stderr.txt"):
             logname = log[:-4]
+            logt = deque([], maxlen)
+            mergelogs[logname] = logt
             with logcollection.open(log) as f:
                 for l in f:
                     if containersapi:
                         g = timestamp_re.match(l)
-                        logtail.append("%s %s %s" % (g.group(1), logname, g.group(2)))
+                        logt.append((g.group(1), g.group(2)))
                     elif not crunchstat_re.match(l):
-                        logtail.append(l)
-    if len(logcollection) > 1:
-        logtail = sorted(logtail)[-maxlen:]
+                        logt.append(l)
+
+    if len(mergelogs) > 1:
+        keys = mergelogs.keys()
+        logtail = []
+        while True:
+            earliest = None
+            for k in keys:
+                if mergelogs[k]:
+                    if earliest is None or mergelogs[k][0][0] < mergelogs[earliest][0][0]:
+                        earliest = k
+            if earliest is None:
+                break
+            ts, msg = mergelogs[earliest].popleft()
+            logtail.append("%s %s %s" % (ts, earliest, msg))
+        logtail = logtail[-maxlen:]
+    else:
+        logtail = mergelogs.values()[0]
+
     logtxt = "\n  ".join(l.strip() for l in logtail)
     logger.info(header)
     logger.info("\n  %s", logtxt)

-----------------------------------------------------------------------


hooks/post-receive
--