[ARVADOS] updated: 1801e182be76647235bf237f614f80ad033528e1
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 9 13:47:46 EST 2016
Summary of changes:
tools/crunchstat-summary/crunchstat_summary/chartjs.py | 9 +++++----
tools/crunchstat-summary/crunchstat_summary/summarizer.py | 2 +-
2 files changed, 6 insertions(+), 5 deletions(-)
discards 7a51378cd74cf33df9d49a2b94d197492caa3ee6 (commit)
discards 1600cdabb1632a672b202150c7b02ef372a86a56 (commit)
discards cf07a2d9d58ff4d5344f19655fc3a37e2281821a (commit)
via 1801e182be76647235bf237f614f80ad033528e1 (commit)
via b04314022bee0447dee03457a453920c505008b1 (commit)
via cacf0b4e45542288adcc83a173f6d48f04b32ab6 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (7a51378cd74cf33df9d49a2b94d197492caa3ee6)
\
N -- N -- N (1801e182be76647235bf237f614f80ad033528e1)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 1801e182be76647235bf237f614f80ad033528e1
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Feb 9 11:49:35 2016 -0500
8341: Include Keep network activity in net stats.
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index a8d3fd0..cf748ff 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -37,8 +37,7 @@ class Summarizer(object):
# stats_max: {category: {stat: val}}
self.stats_max = collections.defaultdict(
- functools.partial(collections.defaultdict,
- lambda: float('-Inf')))
+ functools.partial(collections.defaultdict, lambda: 0))
# task_stats: {task_id: {category: {stat: val}}}
self.task_stats = collections.defaultdict(
functools.partial(collections.defaultdict, dict))
@@ -232,10 +231,12 @@ class Summarizer(object):
self.stats_max['mem']['rss'],
lambda x: x / 1e9),
('Max network traffic in a single task: {}GB',
- self.stats_max['net:eth0']['tx+rx'],
+ self.stats_max['net:eth0']['tx+rx'] +
+ self.stats_max['net:keep0']['tx+rx'],
lambda x: x / 1e9),
('Max network speed in a single interval: {}MB/s',
- self.stats_max['net:eth0']['tx+rx__rate'],
+ self.stats_max['net:eth0']['tx+rx__rate'] +
+ self.stats_max['net:keep0']['tx+rx__rate'],
lambda x: x / 1e6)):
format_string, val, transform = args
if val == float('-Inf'):
commit b04314022bee0447dee03457a453920c505008b1
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Feb 9 10:27:47 2016 -0500
8341: Fix up debug labels. Avoid deadlock after exceptions in thread.
diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index 049b48f..564e820 100644
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -17,6 +17,10 @@ class CollectionReader(object):
"collection {} has {} files; need exactly one".format(
collection_id, len(filenames)))
self._reader = collection.open(filenames[0])
+ self._label = "{}/{}".format(collection_id, filenames[0])
+
+ def __str__(self):
+ return self._label
def __iter__(self):
return iter(self._reader)
@@ -27,33 +31,38 @@ class LiveLogReader(object):
def __init__(self, job_uuid):
logger.debug('load stderr events for job %s', job_uuid)
- self._filters = [
- ['object_uuid', '=', job_uuid],
- ['event_type', '=', 'stderr']]
- self._label = job_uuid
+ self.job_uuid = job_uuid
+
+ def __str__(self):
+ return self.job_uuid
def _get_all_pages(self):
got = 0
last_id = 0
- while True:
- page = arvados.api().logs().index(
- limit=1000,
- order=['id asc'],
- filters=self._filters + [['id','>',str(last_id)]],
- ).execute(num_retries=2)
- got += len(page['items'])
- logger.debug(
- '%s: received %d of %d log events',
- self._label, got,
- got + page['items_available'] - len(page['items']))
- for i in page['items']:
- for line in i['properties']['text'].split('\n'):
- self._queue.put(line+'\n')
- last_id = i['id']
- if (len(page['items']) == 0 or
- len(page['items']) >= page['items_available']):
- break
- self._queue.put(self.EOF)
+ filters = [
+ ['object_uuid', '=', self.job_uuid],
+ ['event_type', '=', 'stderr']]
+ try:
+ while True:
+ page = arvados.api().logs().index(
+ limit=1000,
+ order=['id asc'],
+ filters=filters + [['id','>',str(last_id)]],
+ ).execute(num_retries=2)
+ got += len(page['items'])
+ logger.debug(
+ '%s: received %d of %d log events',
+ self.job_uuid, got,
+ got + page['items_available'] - len(page['items']))
+ for i in page['items']:
+ for line in i['properties']['text'].split('\n'):
+ self._queue.put(line+'\n')
+ last_id = i['id']
+ if (len(page['items']) == 0 or
+ len(page['items']) >= page['items_available']):
+ break
+ finally:
+ self._queue.put(self.EOF)
def __iter__(self):
self._queue = Queue.Queue()
@@ -65,5 +74,6 @@ class LiveLogReader(object):
def next(self):
line = self._queue.get()
if line is self.EOF:
+ self._thread.join()
raise StopIteration
return line
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 70b85f8..a8d3fd0 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -52,10 +52,10 @@ class Summarizer(object):
# constructor will overwrite this with something useful.
self.existing_constraints = {}
- logger.debug("%s: logdata %s", self.label, repr(logdata))
+ logger.debug("%s: logdata %s", self.label, logdata)
def run(self):
- logger.debug("%s: parsing log data", self.label)
+ logger.debug("%s: parsing logdata %s", self.label, self._logdata)
for line in self._logdata:
m = re.search(r'^\S+ \S+ \d+ (?P<seq>\d+) job_task (?P<task_uuid>\S+)$', line)
if m:
commit cacf0b4e45542288adcc83a173f6d48f04b32ab6
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Feb 8 15:47:02 2016 -0500
8341: Do not round up Y axis to even numbers, just use max series value.
Remove Y axis labels (so X axis matches other graphs from the same
job), add grid lines.
diff --git a/tools/crunchstat-summary/crunchstat_summary/chartjs.py b/tools/crunchstat-summary/crunchstat_summary/chartjs.py
index fb30041..03e45e6 100644
--- a/tools/crunchstat-summary/crunchstat_summary/chartjs.py
+++ b/tools/crunchstat-summary/crunchstat_summary/chartjs.py
@@ -2,6 +2,7 @@ from __future__ import print_function
import cgi
import json
+import math
import pkg_resources
from crunchstat_summary import logger
@@ -35,12 +36,25 @@ class ChartJS(object):
}
for s in self.summarizers]
+ def _axisY(self, tasks, stat):
+ ymax = 1
+ for task in tasks.itervalues():
+ for pt in task.series[stat]:
+ ymax = max(ymax, pt[1])
+ ytick = math.exp((1+math.floor(math.log(ymax, 2)))*math.log(2))/4
+ return {
+ 'gridColor': '#cccccc',
+ 'gridThickness': 1,
+ 'interval': ytick,
+ 'minimum': 0,
+ 'maximum': ymax,
+ 'valueFormatString': "''",
+ }
+
def charts(self, label, tasks):
return [
{
- 'axisY': {
- 'minimum': 0,
- },
+ 'axisY': self._axisY(tasks=tasks, stat=stat),
'data': [
{
'type': 'line',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list