[ARVADOS] updated: 7a51378cd74cf33df9d49a2b94d197492caa3ee6
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 9 13:35:13 EST 2016
Summary of changes:
.../crunchstat_summary/reader.py | 41 ++++++++++++----------
.../crunchstat_summary/summarizer.py | 9 ++---
2 files changed, 27 insertions(+), 23 deletions(-)
discards 22607080cb84506ac396e0867192fe8a7723d9e8 (commit)
via 7a51378cd74cf33df9d49a2b94d197492caa3ee6 (commit)
via 1600cdabb1632a672b202150c7b02ef372a86a56 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (22607080cb84506ac396e0867192fe8a7723d9e8)
\
N -- N -- N (7a51378cd74cf33df9d49a2b94d197492caa3ee6)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 7a51378cd74cf33df9d49a2b94d197492caa3ee6
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Feb 9 11:49:35 2016 -0500
8341: Include Keep network activity in net stats.
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 458e2e0..3887d3a 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -37,8 +37,7 @@ class Summarizer(object):
# stats_max: {category: {stat: val}}
self.stats_max = collections.defaultdict(
- functools.partial(collections.defaultdict,
- lambda: float('-Inf')))
+ functools.partial(collections.defaultdict, lambda: 0))
# task_stats: {task_id: {category: {stat: val}}}
self.task_stats = collections.defaultdict(
functools.partial(collections.defaultdict, dict))
@@ -232,10 +231,12 @@ class Summarizer(object):
self.stats_max['mem']['rss'],
lambda x: x / 1e9),
('Max network traffic in a single task: {}GB',
- self.stats_max['net:eth0']['tx+rx'],
+ self.stats_max['net:eth0']['tx+rx'] +
+ self.stats_max['net:keep0']['tx+rx'],
lambda x: x / 1e9),
('Max network speed in a single interval: {}MB/s',
- self.stats_max['net:eth0']['tx+rx__rate'],
+ self.stats_max['net:eth0']['tx+rx__rate'] +
+ self.stats_max['net:keep0']['tx+rx__rate'],
lambda x: x / 1e6)):
format_string, val, transform = args
if val == float('-Inf'):
commit 1600cdabb1632a672b202150c7b02ef372a86a56
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Feb 9 10:27:47 2016 -0500
8341: Fix up debug labels. Avoid deadlock after exceptions in thread.
diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index 049b48f..564e820 100644
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -17,6 +17,10 @@ class CollectionReader(object):
"collection {} has {} files; need exactly one".format(
collection_id, len(filenames)))
self._reader = collection.open(filenames[0])
+ self._label = "{}/{}".format(collection_id, filenames[0])
+
+ def __str__(self):
+ return self._label
def __iter__(self):
return iter(self._reader)
@@ -27,33 +31,38 @@ class LiveLogReader(object):
def __init__(self, job_uuid):
logger.debug('load stderr events for job %s', job_uuid)
- self._filters = [
- ['object_uuid', '=', job_uuid],
- ['event_type', '=', 'stderr']]
- self._label = job_uuid
+ self.job_uuid = job_uuid
+
+ def __str__(self):
+ return self.job_uuid
def _get_all_pages(self):
got = 0
last_id = 0
- while True:
- page = arvados.api().logs().index(
- limit=1000,
- order=['id asc'],
- filters=self._filters + [['id','>',str(last_id)]],
- ).execute(num_retries=2)
- got += len(page['items'])
- logger.debug(
- '%s: received %d of %d log events',
- self._label, got,
- got + page['items_available'] - len(page['items']))
- for i in page['items']:
- for line in i['properties']['text'].split('\n'):
- self._queue.put(line+'\n')
- last_id = i['id']
- if (len(page['items']) == 0 or
- len(page['items']) >= page['items_available']):
- break
- self._queue.put(self.EOF)
+ filters = [
+ ['object_uuid', '=', self.job_uuid],
+ ['event_type', '=', 'stderr']]
+ try:
+ while True:
+ page = arvados.api().logs().index(
+ limit=1000,
+ order=['id asc'],
+ filters=filters + [['id','>',str(last_id)]],
+ ).execute(num_retries=2)
+ got += len(page['items'])
+ logger.debug(
+ '%s: received %d of %d log events',
+ self.job_uuid, got,
+ got + page['items_available'] - len(page['items']))
+ for i in page['items']:
+ for line in i['properties']['text'].split('\n'):
+ self._queue.put(line+'\n')
+ last_id = i['id']
+ if (len(page['items']) == 0 or
+ len(page['items']) >= page['items_available']):
+ break
+ finally:
+ self._queue.put(self.EOF)
def __iter__(self):
self._queue = Queue.Queue()
@@ -65,5 +74,6 @@ class LiveLogReader(object):
def next(self):
line = self._queue.get()
if line is self.EOF:
+ self._thread.join()
raise StopIteration
return line
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 70b85f8..458e2e0 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -52,7 +52,7 @@ class Summarizer(object):
# constructor will overwrite this with something useful.
self.existing_constraints = {}
- logger.debug("%s: logdata %s", self.label, repr(logdata))
+ logger.debug("%s: logdata %s", self.label, logdata)
def run(self):
logger.debug("%s: parsing log data", self.label)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list