[ARVADOS] created: c5618b2b9def604ad7480b01c3b160b576062a09

[Git user]

        at  c5618b2b9def604ad7480b01c3b160b576062a09 (commit)


commit c5618b2b9def604ad7480b01c3b160b576062a09
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jan 12 17:05:23 2017 -0500

    10812: Perform merge of logs based on timestamp instead of sorting log lines.

diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 87908c2..37c408e 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -48,21 +48,44 @@ crunchstat_re = re.compile(r"^\d{4}-\d\d-\d\d_\d\d:\d\d:\d\d [a-z0-9]{5}-8i9sb-[
 timestamp_re = re.compile(r"^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+Z) (.*)")
 
 def logtail(logcollection, logger, header, maxlen=25):
-    logtail = deque([], maxlen*len(logcollection))
+    if len(logcollection) == 0:
+        logger.info(header)
+        logger.info("  ** log is empty **")
+        return
+
     containersapi = ("crunch-run.txt" in logcollection)
+    mergelogs = {}
 
     for log in logcollection.keys():
         if not containersapi or log in ("crunch-run.txt", "stdout.txt", "stderr.txt"):
             logname = log[:-4]
+            logt = deque([], maxlen)
+            mergelogs[logname] = logt
             with logcollection.open(log) as f:
                 for l in f:
                     if containersapi:
                         g = timestamp_re.match(l)
-                        logtail.append("%s %s %s" % (g.group(1), logname, g.group(2)))
+                        logt.append((g.group(1), g.group(2)))
                     elif not crunchstat_re.match(l):
-                        logtail.append(l)
-    if len(logcollection) > 1:
-        logtail = sorted(logtail)[-maxlen:]
+                        logt.append(l)
+
+    if len(mergelogs) > 1:
+        keys = mergelogs.keys()
+        logtail = []
+        while True:
+            earliest = None
+            for k in keys:
+                if mergelogs[k]:
+                    if earliest is None or mergelogs[k][0][0] < mergelogs[earliest][0][0]:
+                        earliest = k
+            if earliest is None:
+                break
+            ts, msg = mergelogs[earliest].popleft()
+            logtail.append("%s %s %s" % (ts, earliest, msg))
+        logtail = logtail[-maxlen:]
+    else:
+        logtail = mergelogs.values()[0]
+
     logtxt = "\n  ".join(l.strip() for l in logtail)
     logger.info(header)
     logger.info("\n  %s", logtxt)

-----------------------------------------------------------------------


hooks/post-receive
--