[ARVADOS] updated: 1.3.0-493-g03fd14e60
Git user
git at public.curoverse.com
Wed Mar 13 19:39:56 EDT 2019
Summary of changes:
build/run-tests.sh | 2 +
.../crunchstat_summary/command.py | 28 +++++++++++++-
.../crunchstat_summary/reader.py | 2 +
.../crunchstat_summary/summarizer.py | 4 +-
...4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report | 2 +-
tools/crunchstat-summary/tests/test_examples.py | 44 +++++++++++++++++-----
6 files changed, 67 insertions(+), 15 deletions(-)
via 03fd14e601b617a456388cbf604463ee98bf93fe (commit)
via aa5db5db07a1b5e9ef059f961f1df047d2469f89 (commit)
via 031a3005dd9ac93a2962a5448a6d77402339a3b2 (commit)
via 22ed89bedb7a8670b8e90dc44d611a6eae87e9a3 (commit)
via cd33e7da1189b1f5a7921b323dbd9cd835420f81 (commit)
from 5d77a55a8989ae60c03bf1bc12c44e17ee531917 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 03fd14e601b617a456388cbf604463ee98bf93fe
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Wed Mar 13 19:33:03 2019 -0400
Add crunchstat-summary to Python 3 tests. refs #14939
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 095d32eaa..c2718fbb6 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -121,6 +121,7 @@ sdk/cwl
sdk/R
tools/sync-groups
tools/crunchstat-summary
+tools/crunchstat-summary:py3
tools/keep-exercise
tools/keep-rsync
tools/keep-block-check
@@ -601,6 +602,7 @@ pythonstuff=(
services/fuse
services/nodemanager
tools/crunchstat-summary
+ tools/crunchstat-summary:py3
)
# If Python 3 is available, set up its virtualenv in $VENV3DIR.
commit aa5db5db07a1b5e9ef059f961f1df047d2469f89
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Wed Mar 13 19:26:00 2019 -0400
Fix test Python 3 integer divide difference. refs #14393
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 16e073011..dba8c6499 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -292,7 +292,7 @@ class Summarizer(object):
self.stats_max['cpu']['user+sys__rate'],
lambda x: x * 100),
('Overall CPU usage: {}%',
- self.job_tot['cpu']['user+sys'] /
+ float(self.job_tot['cpu']['user+sys']) /
self.job_tot['time']['elapsed']
if self.job_tot['time']['elapsed'] > 0 else 0,
lambda x: x * 100),
@@ -346,7 +346,7 @@ class Summarizer(object):
'try runtime_constraints "{}":{}'
).format(
self.label,
- int(math.ceil(cpu_max_rate*100)),
+ math.ceil(cpu_max_rate*100),
constraint_key,
int(used_cores))
diff --git a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
index 981946194..5148a4523 100644
--- a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
+++ b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
@@ -14,7 +14,7 @@ time elapsed 10 - 10
# Number of tasks: 1
# Max CPU time spent by a single task: 0s
# Max CPU usage in a single interval: 0%
-# Overall CPU usage: 0%
+# Overall CPU usage: 0.00%
# Max memory used by a single task: 0.00GB
# Max network traffic in a single task: 0.00GB
# Max network speed in a single interval: 0.00MB/s
commit 031a3005dd9ac93a2962a5448a6d77402339a3b2
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Wed Mar 13 19:03:16 2019 -0400
Fix test file handling to support Python 3. refs #14939
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/tests/test_examples.py b/tools/crunchstat-summary/tests/test_examples.py
index af92becd8..d1351b518 100644
--- a/tools/crunchstat-summary/tests/test_examples.py
+++ b/tools/crunchstat-summary/tests/test_examples.py
@@ -8,6 +8,7 @@ import crunchstat_summary.command
import difflib
import glob
import gzip
+from io import open
import mock
import os
import unittest
@@ -18,10 +19,10 @@ TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
class ReportDiff(unittest.TestCase):
def diff_known_report(self, logfile, cmd):
expectfile = logfile+'.report'
- expect = open(expectfile).readlines()
+ expect = open(expectfile, encoding = 'utf-8').readlines()
self.diff_report(cmd, expect, expectfile=expectfile)
- def diff_report(self, cmd, expect, expectfile=None):
+ def diff_report(self, cmd, expect, expectfile='(expected)'):
got = [x+"\n" for x in cmd.report().strip("\n").split("\n")]
self.assertEqual(got, expect, "\n"+"".join(difflib.context_diff(
expect, got, fromfile=expectfile, tofile="(generated)")))
@@ -54,11 +55,34 @@ class HTMLFromFile(ReportDiff):
class SummarizeEdgeCases(unittest.TestCase):
def test_error_messages(self):
- logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'))
+ logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'), encoding = 'utf-8')
s = crunchstat_summary.summarizer.Summarizer(logfile)
s.run()
+class GzipText(object):
+ def __init__(self, fh):
+ self.fh = fh
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self.fh).decode('utf-8')
+
+ next = __next__
+
+ def close(self):
+ # mimic Gzip behavior and don't close underlying object
+ pass
+
+
class SummarizeContainer(ReportDiff):
fake_container = {
'uuid': '9tee4-dz642-lymtndkpy39eibk',
@@ -89,9 +113,9 @@ class SummarizeContainer(ReportDiff):
'container.json', 'crunchstat.txt', 'arv-mount.txt']
def _open(n):
if n == "crunchstat.txt":
- return gzip.open(self.logfile)
+ return GzipText(gzip.open(self.logfile))
elif n == "arv-mount.txt":
- return gzip.open(self.arvmountlog)
+ return GzipText(gzip.open(self.arvmountlog))
mock_cr().open.side_effect = _open
args = crunchstat_summary.command.ArgumentParser().parse_args(
['--job', self.fake_request['uuid']])
@@ -114,7 +138,7 @@ class SummarizeJob(ReportDiff):
def test_job_report(self, mock_api, mock_cr):
mock_api().jobs().get().execute.return_value = self.fake_job
mock_cr().__iter__.return_value = ['fake-logfile.txt']
- mock_cr().open.return_value = gzip.open(self.logfile)
+ mock_cr().open.return_value = GzipText(gzip.open(self.logfile))
args = crunchstat_summary.command.ArgumentParser().parse_args(
['--job', self.fake_job_uuid])
cmd = crunchstat_summary.command.Command(args)
@@ -175,14 +199,14 @@ class SummarizePipeline(ReportDiff):
mock_api().pipeline_instances().get().execute. \
return_value = self.fake_instance
mock_cr().__iter__.return_value = ['fake-logfile.txt']
- mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
+ mock_cr().open.side_effect = [GzipText(gzip.open(logfile)) for _ in range(3)]
args = crunchstat_summary.command.ArgumentParser().parse_args(
['--pipeline-instance', self.fake_instance['uuid']])
cmd = crunchstat_summary.command.Command(args)
cmd.run()
job_report = [
- line for line in open(logfile+'.report').readlines()
+ line for line in open(logfile+'.report', encoding = 'utf-8').readlines()
if not line.startswith('#!! ')]
expect = (
['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
@@ -251,14 +275,14 @@ class SummarizeACRJob(ReportDiff):
mock_api().jobs().index().execute.return_value = self.fake_jobs_index
mock_api().jobs().get().execute.return_value = self.fake_job
mock_cr().__iter__.return_value = ['fake-logfile.txt']
- mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
+ mock_cr().open.side_effect = [GzipText(gzip.open(logfile)) for _ in range(3)]
args = crunchstat_summary.command.ArgumentParser().parse_args(
['--job', self.fake_job['uuid']])
cmd = crunchstat_summary.command.Command(args)
cmd.run()
job_report = [
- line for line in open(logfile+'.report').readlines()
+ line for line in open(logfile+'.report', encoding='utf-8').readlines()
if not line.startswith('#!! ')]
expect = (
['### Summary for zzzzz-8i9sb-i3e77t9z5y8j9cc (partial) (zzzzz-8i9sb-i3e77t9z5y8j9cc)\n',
commit 22ed89bedb7a8670b8e90dc44d611a6eae87e9a3
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Wed Mar 13 19:02:09 2019 -0400
Make log file reading Python 3 compatible. refs #14939
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/command.py b/tools/crunchstat-summary/crunchstat_summary/command.py
index 71bf38357..6256901e5 100644
--- a/tools/crunchstat-summary/crunchstat_summary/command.py
+++ b/tools/crunchstat-summary/crunchstat_summary/command.py
@@ -4,6 +4,7 @@
import argparse
import gzip
+from io import open
import logging
import sys
@@ -41,6 +42,29 @@ class ArgumentParser(argparse.ArgumentParser):
help='Log more information (once for progress, twice for debug)')
+class GzipText(object):
+ def __init__(self, fh):
+ self.fh = fh
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self.fh).decode('utf-8')
+
+ next = __next__
+
+ def close(self):
+ # mimic Gzip behavior and don't close underlying object
+ pass
+
+
class Command(object):
def __init__(self, args):
self.args = args
@@ -57,9 +81,9 @@ class Command(object):
self.summer = summarizer.NewSummarizer(self.args.job, **kwargs)
elif self.args.log_file:
if self.args.log_file.endswith('.gz'):
- fh = gzip.open(self.args.log_file)
+ fh = GzipText(gzip.open(self.args.log_file))
else:
- fh = open(self.args.log_file)
+ fh = open(self.args.log_file, mode = 'r', encoding = 'utf-8')
self.summer = summarizer.Summarizer(fh, **kwargs)
else:
self.summer = summarizer.Summarizer(sys.stdin, **kwargs)
commit cd33e7da1189b1f5a7921b323dbd9cd835420f81
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date: Wed Mar 13 19:00:44 2019 -0400
Make iterator Python 2 compatible. refs #14939
Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>
diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index 7a763f952..f96b7ee92 100644
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -100,6 +100,8 @@ class LiveLogReader(object):
raise StopIteration
return line
+ next = __next__ # for Python 2
+
def __enter__(self):
return self
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list