[ARVADOS] created: 8f0a83077b24b8826e42e82c06e75c53d8660478
Git user
git at public.curoverse.com
Wed Jan 4 21:58:26 EST 2017
at 8f0a83077b24b8826e42e82c06e75c53d8660478 (commit)
commit 8f0a83077b24b8826e42e82c06e75c53d8660478
Author: Tom Morris <tfmorris at curoverse.com>
Date: Wed Jan 4 21:53:04 2017 -0500
10359: Fix html report. Clean up logging
diff --git a/tools/crunchstat-summary/crunchstat_summary/chartjs.py b/tools/crunchstat-summary/crunchstat_summary/chartjs.py
index 22a2721..acee79c 100644
--- a/tools/crunchstat-summary/crunchstat_summary/chartjs.py
+++ b/tools/crunchstat-summary/crunchstat_summary/chartjs.py
@@ -16,7 +16,7 @@ class ChartJS(object):
<title>{} stats</title>
<script type="text/javascript" src="{}"></script>
<script type="text/javascript">var sections = [
- '''.format(cgi.escape(self.label), self.JSLIB)
+ '''.format(cgi.escape(label), self.JSLIB)
def html_trailer(self):
return '''];\n{}</script>
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 1a51687..46932e8 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -112,7 +112,7 @@ class Summarizer(object):
if m.group('category').endswith(':'):
# "stderr crunchstat: notice: ..."
continue
- elif m.group('category') in ('error', 'caught'):
+ elif m.group('category') in ('error', 'caught', 'Running'):
continue
elif m.group('category') == 'read':
# "stderr crunchstat: read /proc/1234/net/dev: ..."
@@ -180,7 +180,7 @@ class Summarizer(object):
if val > self.stats_max[category][stat]:
self.stats_max[category][stat] = val
except Exception as e:
- logger.info('Skipping malformed line: {}Error was: {}\n'.format(line, e))
+ logger.info('Skipping malformed line beginning: {} Error was: {}\n'.format(line[0:100], e))
logger.debug('%s: done parsing log', self.label)
# Enabling this will roll up stats for all subjobs into the parent job
@@ -417,7 +417,7 @@ class PipelineSummarizer(object):
self.instance = self.arv.pipeline_instances().get(
uuid=pipeline_instance_uuid).execute()
self.summarizers = collections.OrderedDict()
- logger.info("%d total components" % len(self.instance['components']))
+ logger.info("Collecting %d top level components" % len(self.instance['components']))
for cname, component in self.instance['components'].iteritems():
if 'job' not in component:
logger.warning(
@@ -434,6 +434,7 @@ class PipelineSummarizer(object):
summarizer.label = '{} {}'.format(cname, uuid)
self.summarizers[cname] = summarizer
if 'components' in job:
+ logger.debug("Collecting components for %s: job %s", cname, uuid)
for cname, uuid in job['components'].iteritems():
subjob = self.arv.jobs().get(uuid=uuid).execute()
self.summarize_job(cname, subjob, **kwargs)
@@ -446,7 +447,7 @@ class PipelineSummarizer(object):
def text_report(self):
for cname, summarizer in self.summarizers.iteritems():
- logger.info('Running %s', cname)
+ logger.info('Analyzing %s', cname)
summarizer.run()
txt = ''
txt += '### Summary for {} ({})\n'.format(
@@ -456,15 +457,14 @@ class PipelineSummarizer(object):
yield txt
def html_header(self):
- self.chartjs = crunchstat_summary.chartjs.ChartJS(
- self.label)
+ self.chartjs = crunchstat_summary.chartjs.ChartJS()
return self.chartjs.html_header(self.label)
def html_report(self):
- for cname, summarizer in self.summarizers:
- logger.info('Running %s', cname)
+ for cname, summarizer in self.summarizers.iteritems():
+ logger.info('Analyzing %s', cname)
summarizer.run()
- yield self.chartjs.section(summarizer)
+ yield self.chartjs.section(summarizer)+", \n"
def html_trailer(self):
return self.chartjs.html_trailer()
commit d705f9efc6aab2f335bb4eccdca3ae63062b83df
Author: Tom Morris <tfmorris at curoverse.com>
Date: Wed Jan 4 21:06:29 2017 -0500
10359: Refactor to serialize processing
diff --git a/tools/crunchstat-summary/bin/crunchstat-summary b/tools/crunchstat-summary/bin/crunchstat-summary
index e16bd8e..247b794 100755
--- a/tools/crunchstat-summary/bin/crunchstat-summary
+++ b/tools/crunchstat-summary/bin/crunchstat-summary
@@ -12,4 +12,5 @@ logging.getLogger().addHandler(logging.StreamHandler())
args = crunchstat_summary.command.ArgumentParser().parse_args(sys.argv[1:])
cmd = crunchstat_summary.command.Command(args)
cmd.run()
-print(cmd.report(), end='')
+for r in cmd.report():
+ print(r)
diff --git a/tools/crunchstat-summary/crunchstat_summary/chartjs.py b/tools/crunchstat-summary/crunchstat_summary/chartjs.py
index 03e45e6..22a2721 100644
--- a/tools/crunchstat-summary/crunchstat_summary/chartjs.py
+++ b/tools/crunchstat-summary/crunchstat_summary/chartjs.py
@@ -11,30 +11,23 @@ from crunchstat_summary import logger
class ChartJS(object):
JSLIB = 'https://cdnjs.cloudflare.com/ajax/libs/canvasjs/1.7.0/canvasjs.min.js'
- def __init__(self, label, summarizers):
- self.label = label
- self.summarizers = summarizers
-
- def html(self):
+ def html_header(self, label):
return '''<!doctype html><html><head>
<title>{} stats</title>
<script type="text/javascript" src="{}"></script>
- <script type="text/javascript">{}</script>
- </head><body></body></html>
- '''.format(cgi.escape(self.label), self.JSLIB, self.js())
+ <script type="text/javascript">var sections = [
+ '''.format(cgi.escape(self.label), self.JSLIB)
- def js(self):
- return 'var sections = {};\n{}'.format(
- json.dumps(self.sections()),
- pkg_resources.resource_string('crunchstat_summary', 'chartjs.js'))
+ def html_trailer(self):
+ return '''];\n{}</script>
+ </head><body></body></html>
+ '''.format(pkg_resources.resource_string('crunchstat_summary', 'chartjs.js'))
- def sections(self):
- return [
- {
- 'label': s.long_label(),
- 'charts': self.charts(s.label, s.tasks),
- }
- for s in self.summarizers]
+ def section(self, summarizer):
+ return json.dumps({
+ 'label': summarizer.long_label(),
+ 'charts': self.charts(summarizer.label, summarizer.tasks),
+ })
def _axisY(self, tasks, stat):
ymax = 1
diff --git a/tools/crunchstat-summary/crunchstat_summary/command.py b/tools/crunchstat-summary/crunchstat_summary/command.py
index 78638c6..d226dc3 100644
--- a/tools/crunchstat-summary/crunchstat_summary/command.py
+++ b/tools/crunchstat-summary/crunchstat_summary/command.py
@@ -53,10 +53,16 @@ class Command(object):
self.summer = summarizer.Summarizer(fh, **kwargs)
else:
self.summer = summarizer.Summarizer(sys.stdin, **kwargs)
- return self.summer.run()
+ return self.summer
def report(self):
- if self.args.format == 'html':
- return self.summer.html_report()
- elif self.args.format == 'text':
- return self.summer.text_report()
+ if self.args.format == 'text' or self.args.format == 'both':
+ yield self.summer.text_header()
+ for r in self.summer.text_report():
+ yield r
+ yield self.summer.text_trailer()
+ if self.args.format == 'html' or self.args.format == 'both':
+ yield self.summer.html_header()
+ for r in self.summer.html_report():
+ yield r
+ yield self.summer.html_trailer()
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index f1c97c2..1a51687 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -10,7 +10,6 @@ import itertools
import math
import re
import sys
-import threading
import _strptime
from arvados.api import OrderedJsonModel
@@ -415,16 +414,18 @@ class JobSummarizer(Summarizer):
class PipelineSummarizer(object):
def __init__(self, pipeline_instance_uuid, **kwargs):
self.arv = arvados.api('v1', model=OrderedJsonModel())
- instance = self.arv.pipeline_instances().get(
+ self.instance = self.arv.pipeline_instances().get(
uuid=pipeline_instance_uuid).execute()
self.summarizers = collections.OrderedDict()
- for cname, component in instance['components'].iteritems():
+ logger.info("%d total components" % len(self.instance['components']))
+ for cname, component in self.instance['components'].iteritems():
if 'job' not in component:
logger.warning(
"%s: skipping component with no job assigned", cname)
else:
self.summarize_job(cname, component['job'], **kwargs)
self.label = pipeline_instance_uuid
+ self._kwargs = kwargs
def summarize_job(self, cname, job, **kwargs):
uuid = job['uuid']
@@ -437,25 +438,33 @@ class PipelineSummarizer(object):
subjob = self.arv.jobs().get(uuid=uuid).execute()
self.summarize_job(cname, subjob, **kwargs)
- def run(self):
- threads = []
- for summarizer in self.summarizers.itervalues():
- t = threading.Thread(target=summarizer.run)
- t.daemon = True
- t.start()
- threads.append(t)
- for t in threads:
- t.join()
+ def text_header(self):
+ return ''
+
+ def text_trailer(self):
+ return ''
def text_report(self):
- txt = ''
for cname, summarizer in self.summarizers.iteritems():
+ logger.info('Running %s', cname)
+ summarizer.run()
+ txt = ''
txt += '### Summary for {} ({})\n'.format(
cname, summarizer.job['uuid'])
txt += summarizer.text_report()
txt += '\n'
- return txt
+ yield txt
+
+ def html_header(self):
+ self.chartjs = crunchstat_summary.chartjs.ChartJS(
+ self.label)
+ return self.chartjs.html_header(self.label)
def html_report(self):
- return crunchstat_summary.chartjs.ChartJS(
- self.label, self.summarizers.itervalues()).html()
+ for cname, summarizer in self.summarizers:
+ logger.info('Running %s', cname)
+ summarizer.run()
+ yield self.chartjs.section(summarizer)
+
+ def html_trailer(self):
+ return self.chartjs.html_trailer()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list