[ARVADOS] created: 1.3.0-1275-g7a42fbd26
Git user
git at public.curoverse.com
Tue Jul 9 18:25:28 UTC 2019
at 7a42fbd26f8bf162fdee43b3cfff8950d57ed203 (commit)
commit 7a42fbd26f8bf162fdee43b3cfff8950d57ed203
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Tue Jul 9 14:04:41 2019 -0400
Fix up test gold standard references for new output
refs #13913
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 305042b2b..e962ced31 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -443,9 +443,9 @@ class Summarizer(object):
def _recommend_temp_disk(self):
"""Recommend decreasing temp disk if utilization < 50%"""
total = float(self.job_tot['statfs']['total'])
- utilization = (float(self.job_tot['statfs']['used']) / total)
+ utilization = (float(self.job_tot['statfs']['used']) / total) if total > 0 else 0.0
- if utilization < 50.8:
+ if utilization < 50.8 and total > 0:
yield (
'#!! {} max temp disk utilization was {:.0f}% of {:.0f} MiB -- '
'consider reducing "tmpdirMin" and/or "outdirMin"'
diff --git a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
index 0691e4f1e..f77059b82 100644
--- a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
+++ b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
@@ -20,3 +20,4 @@ time elapsed 10 - 10
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
diff --git a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz.report b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz.report
index c64c34c80..b17c70059 100644
--- a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz.report
+++ b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-crunchstat.txt.gz.report
@@ -23,4 +23,6 @@ time elapsed 20 - 20
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
-#!! container max RSS was 67 MiB -- try reducing runtime_constraints to "ram":1020054732
+# Temp disk utilization 1.21%
+#!! label #1 max RSS was 67 MiB -- try reducing runtime_constraints to "ram":1020054732
+#!! label #1 max temp disk utilization was 1% of 383960 MiB -- consider reducing "tmpdirMin" and/or "outdirMin"
diff --git a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk.txt.gz.report b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk.txt.gz.report
index 3075c24b9..5152e577f 100644
--- a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk.txt.gz.report
+++ b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk.txt.gz.report
@@ -34,4 +34,6 @@ time elapsed 20 - 20
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 1.21%
#!! container max RSS was 67 MiB -- try reducing runtime_constraints to "ram":1020054732
+#!! container max temp disk utilization was 1% of 383960 MiB -- consider reducing "tmpdirMin" and/or "outdirMin"
diff --git a/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report b/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
index 5e3ad152f..1fb56c7be 100644
--- a/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
+++ b/tools/crunchstat-summary/tests/logfile_20151204190335.txt.gz.report
@@ -31,4 +31,5 @@ time elapsed 80 - 80
# Max network speed in a single interval: 42.58MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
#!! 4xphq-8i9sb-jq0ekny1xou3zoh max RSS was 334 MiB -- try reducing runtime_constraints to "min_ram_mb_per_node":972
diff --git a/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report b/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
index e260ca5bd..f567233fb 100644
--- a/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
+++ b/tools/crunchstat-summary/tests/logfile_20151210063411.txt.gz.report
@@ -20,4 +20,5 @@ time elapsed 2 - 4
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
#!! 4xphq-8i9sb-zvb2ocfycpomrup max RSS was 1 MiB -- try reducing runtime_constraints to "min_ram_mb_per_node":972
diff --git a/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report b/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
index ffe107225..ab0febbef 100644
--- a/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
+++ b/tools/crunchstat-summary/tests/logfile_20151210063439.txt.gz.report
@@ -20,4 +20,5 @@ time elapsed 2 - 3
# Max network speed in a single interval: 0.00MB/s
# Keep cache miss rate 0.00%
# Keep cache utilization 0.00%
+# Temp disk utilization 0.00%
#!! 4xphq-8i9sb-v831jm2uq0g2g9x max RSS was 1 MiB -- try reducing runtime_constraints to "min_ram_mb_per_node":972
diff --git a/tools/crunchstat-summary/tests/test_examples.py b/tools/crunchstat-summary/tests/test_examples.py
index 7603ea488..0270eaaec 100644
--- a/tools/crunchstat-summary/tests/test_examples.py
+++ b/tools/crunchstat-summary/tests/test_examples.py
@@ -102,7 +102,7 @@ class SummarizeContainer(ReportDiff):
return UTF8Decode(gzip.open(self.arvmountlog))
mock_cr().open.side_effect = _open
args = crunchstat_summary.command.ArgumentParser().parse_args(
- ['--job', self.fake_request['uuid']])
+ ['--container', self.fake_request['uuid']])
cmd = crunchstat_summary.command.Command(args)
cmd.run()
self.diff_known_report(self.reportfile, cmd)
commit c338786f0cbcdbf2e992c96eccc26a67938095f0
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Wed Jul 3 13:52:34 2019 -0400
Add a simple temp disk utilization based recommendation.
refs #13913
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index a86702ed7..305042b2b 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -335,7 +335,9 @@ class Summarizer(object):
return itertools.chain(
self._recommend_cpu(),
self._recommend_ram(),
- self._recommend_keep_cache())
+ self._recommend_keep_cache(),
+ self._recommend_temp_disk(),
+ )
def _recommend_cpu(self):
"""Recommend asking for 4 cores if max CPU usage was 333%"""
@@ -438,6 +440,21 @@ class Summarizer(object):
math.ceil(asked_cache * 2 / self._runtime_constraint_mem_unit()))
+ def _recommend_temp_disk(self):
+ """Recommend decreasing temp disk if utilization < 50%"""
+ total = float(self.job_tot['statfs']['total'])
+ utilization = (float(self.job_tot['statfs']['used']) / total)
+
+ if utilization < 50.8:
+ yield (
+ '#!! {} max temp disk utilization was {:.0f}% of {:.0f} MiB -- '
+ 'consider reducing "tmpdirMin" and/or "outdirMin"'
+ ).format(
+ self.label,
+ utilization * 100.0,
+ total / MB)
+
+
def _format(self, val):
"""Return a string representation of a stat.
commit 62d7b55931373e5c212f557a642e07a64ca923cd
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Mon Apr 22 20:48:53 2019 -0400
Graph temp disk usage
Plots temp disk space used as well as total available.
Also adds support for multiple metrics per graph and
groups CPU and network stats by category.
refs #13913
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/dygraphs.py b/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
index 6df440a14..10f1f26f4 100644
--- a/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
+++ b/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
@@ -13,21 +13,34 @@ class DygraphsChart(crunchstat_summary.webchart.WebChart):
def headHTML(self):
return '<link rel="stylesheet" href="{}">\n'.format(self.CSS)
- def chartdata(self, label, tasks, stat):
+ def chartdata(self, label, tasks, stats):
+ '''For Crunch2, label is the name of container request,
+ tasks is the top level container and
+ stats is index by a tuple of (category, metric).
+ '''
return {
- 'data': self._collate_data(tasks, stat),
+ 'data': self._collate_data(tasks, stats),
'options': {
+ 'legend': 'always',
'connectSeparatedPoints': True,
- 'labels': ['elapsed']+[uuid for uuid, _ in tasks.items()],
- 'title': '{}: {} {}'.format(label, stat[0], stat[1]),
+ 'labels': ['elapsed'] + stats[1],
+ 'title': '{}: {}'.format(label, stats[0]),
},
}
- def _collate_data(self, tasks, stat):
+ def _collate_data(self, tasks, stats):
data = []
nulls = []
+ # uuid is category for crunch2
for uuid, task in tasks.items():
- for pt in task.series[stat]:
- data.append([pt[0].total_seconds()] + nulls + [pt[1]])
+ # All stats in a category are assumed to have the same time base and same number of samples
+ category = stats[0]
+ series_names = stats[1]
+ sn0 = series_names[0]
+ series = task.series[(category,sn0)]
+ for i in range(len(series)):
+ pt = series[i]
+ vals = [task.series[(category,stat)][i][1] for stat in series_names[1:]]
+ data.append([pt[0].total_seconds()] + nulls + [pt[1]] + vals)
nulls.append(None)
return sorted(data)
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 884f16b4a..a86702ed7 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -129,13 +129,14 @@ class Summarizer(object):
try:
self.label = m.group('job_uuid')
except IndexError:
- self.label = 'container'
- if m.group('category').endswith(':'):
+ self.label = 'label #1'
+ category = m.group('category')
+ if category.endswith(':'):
# "stderr crunchstat: notice: ..."
continue
- elif m.group('category') in ('error', 'caught'):
+ elif category in ('error', 'caught'):
continue
- elif m.group('category') in ('read', 'open', 'cgroup', 'CID', 'Running'):
+ elif category in ('read', 'open', 'cgroup', 'CID', 'Running'):
# "stderr crunchstat: read /proc/1234/net/dev: ..."
# (old logs are less careful with unprefixed error messages)
continue
@@ -221,11 +222,11 @@ class Summarizer(object):
if group == 'interval' and this_interval_s:
stat = stat + '__rate'
val = val / this_interval_s
- if stat in ['user+sys__rate', 'tx+rx__rate']:
+ if stat in ['user+sys__rate', 'user__rate', 'sys__rate', 'tx+rx__rate', 'rx__rate', 'tx__rate']:
task.series[category, stat].append(
(timestamp - self.starttime, val))
else:
- if stat in ['rss']:
+ if stat in ['rss','used','total']:
task.series[category, stat].append(
(timestamp - self.starttime, val))
self.task_stats[task_id][category][stat] = val
@@ -315,7 +316,13 @@ class Summarizer(object):
(float(self.job_tot['blkio:0:0']['read']) /
float(self.job_tot['net:keep0']['rx']))
if self.job_tot['net:keep0']['rx'] > 0 else 0,
- lambda x: x * 100.0)):
+ lambda x: x * 100.0),
+ ('Temp disk utilization {}%',
+ (float(self.job_tot['statfs']['used']) /
+ float(self.job_tot['statfs']['total']))
+ if self.job_tot['statfs']['total'] > 0 else 0,
+ lambda x: x * 100.0),
+ ):
format_string, val, transform = args
if val == float('-Inf'):
continue
diff --git a/tools/crunchstat-summary/crunchstat_summary/webchart.py b/tools/crunchstat-summary/crunchstat_summary/webchart.py
index cf0c1e67a..31afcf64e 100644
--- a/tools/crunchstat-summary/crunchstat_summary/webchart.py
+++ b/tools/crunchstat-summary/crunchstat_summary/webchart.py
@@ -45,10 +45,13 @@ class WebChart(object):
'label': s.long_label(),
'charts': [
self.chartdata(s.label, s.tasks, stat)
- for stat in (('cpu', 'user+sys__rate'),
- ('mem', 'rss'),
- ('net:eth0', 'tx+rx__rate'),
- ('net:keep0', 'tx+rx__rate'))],
+ for stat in (('cpu', ['user+sys__rate', 'user__rate', 'sys__rate']),
+ ('mem', ['rss']),
+ ('net:eth0', ['tx+rx__rate','rx__rate','tx__rate']),
+ ('net:keep0', ['tx+rx__rate','rx__rate','tx__rate']),
+ ('statfs', ['used', 'total']),
+ )
+ ],
}
for s in self.summarizers]
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list