[ARVADOS] updated: b03a6a8b2c20a0579cd724baeb9283bd5f0d1f08

[git at public.curoverse.com]

Summary of changes:
 services/api/script/crunch-failure-report.py | 38 +++++++++++++++-------------
 1 file changed, 21 insertions(+), 17 deletions(-)

       via  b03a6a8b2c20a0579cd724baeb9283bd5f0d1f08 (commit)
      from  0dd15acb9a97a73d1fc06eccbe24f18d28f473c5 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit b03a6a8b2c20a0579cd724baeb9283bd5f0d1f08
Author: Tim Pierce <twp at curoverse.com>
Date:   Fri Dec 12 11:22:07 2014 -0500

    4598: correct search criteria for Crunch job output
    
    Crunch stderr output is stored in the properties/text field of logs with
    event_type 'stderr'.
    
    Limited the filter criteria to event_type = 'stderr' and modified
    text pattern filtering appropriately.
    
    Now uses arvados.util.list_all to retrieve all stderr rows within the
    specified time frame.

diff --git a/services/api/script/crunch-failure-report.py b/services/api/script/crunch-failure-report.py
index 5ba34c3..e2b5d2d 100755
--- a/services/api/script/crunch-failure-report.py
+++ b/services/api/script/crunch-failure-report.py
@@ -3,7 +3,7 @@
 import argparse
 import datetime
 import json
-import pprint
+import re
 import sys
 
 import arvados
@@ -19,10 +19,9 @@ def parse_arguments(arguments):
         '--end',
         help='End date and time')
     arg_parser.add_argument(
-        '--summary',
-        action='append',
-        default=[],
-        help='SQL pattern (ILIKE syntax) to match on summary lines')
+        '--match',
+        default='fail',
+        help='Regular expression to match on Crunch error output lines.')
 
     return arg_parser.parse_args(arguments)
 
@@ -43,21 +42,26 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
     now = datetime.datetime.utcnow()
     start_time = args.start or api_timestamp(now - datetime.timedelta(days=1))
     end_time = args.end or api_timestamp(now)
+    match_re = re.compile(args.match)
 
-    summary_patterns = args.summary or ['%fail%']
-    logfilters = [['summary', 'ilike', pattern] for pattern in summary_patterns ]
-    logfilters.append( ['created_at', '>=', start_time] )
-    logfilters.append( ['created_at', '<=', end_time] )
-
-    logs = api.logs().list(
-        filters=json.dumps(logfilters)
-    ).execute()
+    logs = arvados.util.list_all(
+        api.logs().list,
+        filters=json.dumps([ ['created_at', '>=', start_time],
+                             ['created_at', '<=', end_time],
+                             ['event_type', '=', 'stderr'] ]))
 
     log_stats = {}
-    for log in logs['items']:
-        summary = log['summary']
-        log_uuid = log['uuid']
-        log_stats.setdefault(summary, []).append(log_uuid)
+    for log in logs:
+        for logline in log['properties']['text'].splitlines():
+            # Remove timestamp at beginning of log lines
+            # Mon Dec  1 23:59:55 2014
+            stderr_msg = re.sub(
+                '\w{3} \w{3} +\d+ \d{2}:\d{2}:\d{2} \d{4} +',
+                '',
+                logline)
+            if match_re.search(stderr_msg):
+                log_uuid = log['uuid']
+                log_stats.setdefault(stderr_msg, []).append(log_uuid)
 
     # Sort the keys of log stats in decreasing order of frequency.
     for k in sorted(log_stats.keys(), cmp=lambda a,b: cmp(len(log_stats[b]), len(log_stats[a]))):

-----------------------------------------------------------------------


hooks/post-receive
--