[ARVADOS] created: 1.3.0-488-g5d77a55a8
Git user
git at public.curoverse.com
Fri Mar 8 18:41:16 EST 2019
at 5d77a55a8989ae60c03bf1bc12c44e17ee531917 (commit)
commit 5d77a55a8989ae60c03bf1bc12c44e17ee531917
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Fri Mar 8 18:40:24 2019 -0500
Update crunchstat-summary to Python 3. Refs #14939
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/dygraphs.py b/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
index 1314e9df3..6df440a14 100644
--- a/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
+++ b/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
@@ -18,7 +18,7 @@ class DygraphsChart(crunchstat_summary.webchart.WebChart):
'data': self._collate_data(tasks, stat),
'options': {
'connectSeparatedPoints': True,
- 'labels': ['elapsed']+[uuid for uuid, _ in tasks.iteritems()],
+ 'labels': ['elapsed']+[uuid for uuid, _ in tasks.items()],
'title': '{}: {} {}'.format(label, stat[0], stat[1]),
},
}
@@ -26,7 +26,7 @@ class DygraphsChart(crunchstat_summary.webchart.WebChart):
def _collate_data(self, tasks, stat):
data = []
nulls = []
- for uuid, task in tasks.iteritems():
+ for uuid, task in tasks.items():
for pt in task.series[stat]:
data.append([pt[0].total_seconds()] + nulls + [pt[1]])
nulls.append(None)
diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index 311c006c0..7a763f952 100644
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -2,11 +2,11 @@
#
# SPDX-License-Identifier: AGPL-3.0
-from __future__ import print_function
+
import arvados
import itertools
-import Queue
+import queue
import threading
from crunchstat_summary import logger
@@ -87,13 +87,13 @@ class LiveLogReader(object):
self._queue.put(self.EOF)
def __iter__(self):
- self._queue = Queue.Queue()
+ self._queue = queue.Queue()
self._thread = threading.Thread(target=self._get_all_pages)
self._thread.daemon = True
self._thread.start()
return self
- def next(self):
+ def __next__(self):
line = self._queue.get()
if line is self.EOF:
self._thread.join()
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index b2f6f1bb7..16e073011 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -2,7 +2,7 @@
#
# SPDX-License-Identifier: AGPL-3.0
-from __future__ import print_function
+
import arvados
import collections
@@ -209,7 +209,7 @@ class Summarizer(object):
stats['user+sys'] = stats.get('user', 0) + stats.get('sys', 0)
if 'tx' in stats or 'rx' in stats:
stats['tx+rx'] = stats.get('tx', 0) + stats.get('rx', 0)
- for stat, val in stats.iteritems():
+ for stat, val in stats.items():
if group == 'interval':
if stat == 'seconds':
this_interval_s = val
@@ -236,9 +236,9 @@ class Summarizer(object):
self.job_tot = collections.defaultdict(
functools.partial(collections.defaultdict, int))
- for task_id, task_stat in self.task_stats.iteritems():
- for category, stat_last in task_stat.iteritems():
- for stat, val in stat_last.iteritems():
+ for task_id, task_stat in self.task_stats.items():
+ for category, stat_last in task_stat.items():
+ for stat, val in stat_last.items():
if stat in ['cpus', 'cache', 'swap', 'rss']:
# meaningless stats like 16 cpu cores x 5 tasks = 80
continue
@@ -273,8 +273,8 @@ class Summarizer(object):
def _text_report_gen(self):
yield "\t".join(['category', 'metric', 'task_max', 'task_max_rate', 'job_total'])
- for category, stat_max in sorted(self.stats_max.iteritems()):
- for stat, val in sorted(stat_max.iteritems()):
+ for category, stat_max in sorted(self.stats_max.items()):
+ for stat, val in sorted(stat_max.items()):
if stat.endswith('__rate'):
continue
max_rate = self._format(stat_max.get(stat+'__rate', '-'))
@@ -539,7 +539,7 @@ class MultiSummarizer(object):
def run(self):
threads = []
- for child in self.children.itervalues():
+ for child in self.children.values():
self.throttle.acquire()
t = threading.Thread(target=self.run_and_release, args=(child.run, ))
t.daemon = True
@@ -551,7 +551,7 @@ class MultiSummarizer(object):
def text_report(self):
txt = ''
d = self._descendants()
- for child in d.itervalues():
+ for child in d.values():
if len(d) > 1:
txt += '### Summary for {} ({})\n'.format(
child.label, child.process['uuid'])
@@ -566,7 +566,7 @@ class MultiSummarizer(object):
MultiSummarizers) are omitted.
"""
d = collections.OrderedDict()
- for key, child in self.children.iteritems():
+ for key, child in self.children.items():
if isinstance(child, Summarizer):
d[key] = child
if isinstance(child, MultiSummarizer):
@@ -574,7 +574,7 @@ class MultiSummarizer(object):
return d
def html_report(self):
- return WEBCHART_CLASS(self.label, self._descendants().itervalues()).html()
+ return WEBCHART_CLASS(self.label, iter(self._descendants().values())).html()
class JobTreeSummarizer(MultiSummarizer):
@@ -588,7 +588,7 @@ class JobTreeSummarizer(MultiSummarizer):
preloaded = {}
for j in arv.jobs().index(
limit=len(job['components']),
- filters=[['uuid','in',job['components'].values()]]).execute()['items']:
+ filters=[['uuid','in',list(job['components'].values())]]).execute()['items']:
preloaded[j['uuid']] = j
for cname in sorted(job['components'].keys()):
child_uuid = job['components'][cname]
@@ -605,7 +605,7 @@ class JobTreeSummarizer(MultiSummarizer):
class PipelineSummarizer(MultiSummarizer):
def __init__(self, instance, **kwargs):
children = collections.OrderedDict()
- for cname, component in instance['components'].iteritems():
+ for cname, component in instance['components'].items():
if 'job' not in component:
logger.warning(
"%s: skipping component with no job assigned", cname)
@@ -663,7 +663,7 @@ class ContainerTreeSummarizer(MultiSummarizer):
cr['name'] = cr.get('name') or cr['uuid']
todo.append(cr)
sorted_children = collections.OrderedDict()
- for uuid in sorted(children.keys(), key=lambda uuid: children[uuid].sort_key):
+ for uuid in sorted(list(children.keys()), key=lambda uuid: children[uuid].sort_key):
sorted_children[uuid] = children[uuid]
super(ContainerTreeSummarizer, self).__init__(
children=sorted_children,
diff --git a/tools/crunchstat-summary/crunchstat_summary/webchart.py b/tools/crunchstat-summary/crunchstat_summary/webchart.py
index 9d18883ce..376696c0b 100644
--- a/tools/crunchstat-summary/crunchstat_summary/webchart.py
+++ b/tools/crunchstat-summary/crunchstat_summary/webchart.py
@@ -21,7 +21,7 @@ class WebChart(object):
self.summarizers = summarizers
def html(self):
- return '''<!doctype html><html><head>
+ return u'''<!doctype html><html><head>
<title>{} stats</title>
<script type="text/javascript" src="{}"></script>
<script type="text/javascript">{}</script>
@@ -31,20 +31,20 @@ class WebChart(object):
self.JSLIB, self.js(), self.headHTML())
def js(self):
- return 'var chartdata = {};\n{}'.format(
+ return u'var chartdata = {};\n{}'.format(
json.dumps(self.sections()),
- '\n'.join([pkg_resources.resource_string('crunchstat_summary', jsa) for jsa in self.JSASSETS]))
+ u'\n'.join([pkg_resources.resource_string(u'crunchstat_summary', jsa).decode('utf-8') for jsa in self.JSASSETS]))
def sections(self):
return [
{
- 'label': s.long_label(),
- 'charts': [
+ u'label': s.long_label(),
+ u'charts': [
self.chartdata(s.label, s.tasks, stat)
- for stat in (('cpu', 'user+sys__rate'),
- ('mem', 'rss'),
- ('net:eth0', 'tx+rx__rate'),
- ('net:keep0', 'tx+rx__rate'))],
+ for stat in ((u'cpu', u'user+sys__rate'),
+ (u'mem', u'rss'),
+ (u'net:eth0', u'tx+rx__rate'),
+ (u'net:keep0', u'tx+rx__rate'))],
}
for s in self.summarizers]
@@ -58,4 +58,4 @@ class WebChart(object):
def headHTML(self):
"""Return extra HTML text to include in HEAD."""
- return ''
+ return u''
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list