[ARVADOS] created: 1.3.0-527-ga0c6c7f80

Git user git at public.curoverse.com
Thu Mar 14 18:56:58 EDT 2019


        at  a0c6c7f8037dd1265f85bba49a7e6ac84a4253eb (commit)


commit a0c6c7f8037dd1265f85bba49a7e6ac84a4253eb
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date:   Thu Mar 14 18:47:20 2019 -0400

    Resolve deprecation warnings
    
    Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>

diff --git a/tools/crunchstat-summary/crunchstat_summary/webchart.py b/tools/crunchstat-summary/crunchstat_summary/webchart.py
index 6931937b0..f7017e84d 100644
--- a/tools/crunchstat-summary/crunchstat_summary/webchart.py
+++ b/tools/crunchstat-summary/crunchstat_summary/webchart.py
@@ -2,7 +2,11 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-import cgi
+try:
+    from html import escape
+except ImportError:
+    from cgi import escape
+
 import json
 import pkg_resources
 
@@ -27,7 +31,7 @@ class WebChart(object):
         <script type="text/javascript">{}</script>
         {}
         </head><body></body></html>
-        '''.format(cgi.escape(self.label),
+        '''.format(escape(self.label),
                    self.JSLIB, self.js(), self.headHTML())
 
     def js(self):
diff --git a/tools/crunchstat-summary/tests/test_examples.py b/tools/crunchstat-summary/tests/test_examples.py
index ddec0df3e..0e5d15e9f 100644
--- a/tools/crunchstat-summary/tests/test_examples.py
+++ b/tools/crunchstat-summary/tests/test_examples.py
@@ -11,6 +11,7 @@ import gzip
 from io import open
 import mock
 import os
+import sys
 import unittest
 
 from crunchstat_summary.command import UTF8Decode
@@ -53,7 +54,11 @@ class HTMLFromFile(ReportDiff):
                 ['--format=html', '--log-file', logfile])
             cmd = crunchstat_summary.command.Command(args)
             cmd.run()
-            self.assertRegexpMatches(cmd.report(), r'(?is)<html>.*</html>\s*$')
+            if sys.version_info >= (3,2):
+                self.assertRegex(cmd.report(), r'(?is)<html>.*</html>\s*$')
+            else:
+                self.assertRegexpMatches(cmd.report(), r'(?is)<html>.*</html>\s*$')
+
 
 
 class SummarizeEdgeCases(unittest.TestCase):

commit c3df4f11cdc70d160c51150ede096fdd1ac3fb04
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date:   Thu Mar 14 18:30:52 2019 -0400

    Fix resource leak warned by Python 3
    
    Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>

diff --git a/tools/crunchstat-summary/tests/test_examples.py b/tools/crunchstat-summary/tests/test_examples.py
index dd57cda2d..ddec0df3e 100644
--- a/tools/crunchstat-summary/tests/test_examples.py
+++ b/tools/crunchstat-summary/tests/test_examples.py
@@ -21,7 +21,8 @@ TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
 class ReportDiff(unittest.TestCase):
     def diff_known_report(self, logfile, cmd):
         expectfile = logfile+'.report'
-        expect = open(expectfile, encoding='utf-8').readlines()
+        with open(expectfile, encoding='utf-8') as f:
+            expect = f.readlines()
         self.diff_report(cmd, expect, expectfile=expectfile)
 
     def diff_report(self, cmd, expect, expectfile='(expected)'):
@@ -184,9 +185,8 @@ class SummarizePipeline(ReportDiff):
         cmd = crunchstat_summary.command.Command(args)
         cmd.run()
 
-        job_report = [
-            line for line in open(logfile+'.report', encoding='utf-8').readlines()
-            if not line.startswith('#!! ')]
+        with open(logfile+'.report', encoding='utf-8') as f:
+            job_report = [line for line in f if not line.startswith('#!! ')]
         expect = (
             ['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
             job_report + ['\n'] +
@@ -260,9 +260,8 @@ class SummarizeACRJob(ReportDiff):
         cmd = crunchstat_summary.command.Command(args)
         cmd.run()
 
-        job_report = [
-            line for line in open(logfile+'.report', encoding='utf-8').readlines()
-            if not line.startswith('#!! ')]
+        with open(logfile+'.report', encoding='utf-8') as f:
+            job_report = [line for line in f if not line.startswith('#!! ')]
         expect = (
             ['### Summary for zzzzz-8i9sb-i3e77t9z5y8j9cc (partial) (zzzzz-8i9sb-i3e77t9z5y8j9cc)\n',
              '(no report generated)\n',

commit c1589536caa10777272ced738d35d95a4753c214
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date:   Wed Mar 13 19:33:03 2019 -0400

    Add crunchstat-summary to Python 3 tests. refs #14939
    
    Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 095d32eaa..c2718fbb6 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -121,6 +121,7 @@ sdk/cwl
 sdk/R
 tools/sync-groups
 tools/crunchstat-summary
+tools/crunchstat-summary:py3
 tools/keep-exercise
 tools/keep-rsync
 tools/keep-block-check
@@ -601,6 +602,7 @@ pythonstuff=(
     services/fuse
     services/nodemanager
     tools/crunchstat-summary
+    tools/crunchstat-summary:py3
     )
 
 # If Python 3 is available, set up its virtualenv in $VENV3DIR.

commit 77d60f9088f44baebc2258f1048104831fad24c5
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date:   Wed Mar 13 19:26:00 2019 -0400

    Fix test Python 3 integer divide difference.
    
    Use float for both Python 2 & 3 like the rest of the metrics.
    refs #14393
    
    Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>

diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index 16e073011..dba8c6499 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -292,7 +292,7 @@ class Summarizer(object):
                  self.stats_max['cpu']['user+sys__rate'],
                  lambda x: x * 100),
                 ('Overall CPU usage: {}%',
-                 self.job_tot['cpu']['user+sys'] /
+                 float(self.job_tot['cpu']['user+sys']) /
                  self.job_tot['time']['elapsed']
                  if self.job_tot['time']['elapsed'] > 0 else 0,
                  lambda x: x * 100),
@@ -346,7 +346,7 @@ class Summarizer(object):
                 'try runtime_constraints "{}":{}'
             ).format(
                 self.label,
-                int(math.ceil(cpu_max_rate*100)),
+                math.ceil(cpu_max_rate*100),
                 constraint_key,
                 int(used_cores))
 
diff --git a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
index 981946194..5148a4523 100644
--- a/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
+++ b/tools/crunchstat-summary/tests/container_9tee4-dz642-lymtndkpy39eibk-arv-mount.txt.gz.report
@@ -14,7 +14,7 @@ time	elapsed	10	-	10
 # Number of tasks: 1
 # Max CPU time spent by a single task: 0s
 # Max CPU usage in a single interval: 0%
-# Overall CPU usage: 0%
+# Overall CPU usage: 0.00%
 # Max memory used by a single task: 0.00GB
 # Max network traffic in a single task: 0.00GB
 # Max network speed in a single interval: 0.00MB/s

commit 6bac55d24cdda703cb3c68cef5023bd13844bd03
Author: Tom Morris <tfmorris at veritasgenetics.com>
Date:   Fri Mar 8 18:40:24 2019 -0500

    Update crunchstat-summary to Python 3. Refs #14939
    
    Arvados-DCO-1.1-Signed-off-by: Tom Morris <tfmorris at veritasgenetics.com>

diff --git a/tools/crunchstat-summary/crunchstat_summary/command.py b/tools/crunchstat-summary/crunchstat_summary/command.py
index 71bf38357..aadc77582 100644
--- a/tools/crunchstat-summary/crunchstat_summary/command.py
+++ b/tools/crunchstat-summary/crunchstat_summary/command.py
@@ -4,6 +4,7 @@
 
 import argparse
 import gzip
+from io import open
 import logging
 import sys
 
@@ -41,6 +42,31 @@ class ArgumentParser(argparse.ArgumentParser):
             help='Log more information (once for progress, twice for debug)')
 
 
+class UTF8Decode(object):
+    '''Wrap a file-like iterable to decode UTF-8 bytes into a strings
+    '''
+    def __init__(self, fh):
+        self.fh = fh
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        return next(self.fh).decode('utf-8')
+
+    next = __next__
+
+    def close(self):
+        # mimic Gzip behavior and don't close underlying object
+        pass
+
+
 class Command(object):
     def __init__(self, args):
         self.args = args
@@ -57,9 +83,9 @@ class Command(object):
             self.summer = summarizer.NewSummarizer(self.args.job, **kwargs)
         elif self.args.log_file:
             if self.args.log_file.endswith('.gz'):
-                fh = gzip.open(self.args.log_file)
+                fh = UTF8Decode(gzip.open(self.args.log_file))
             else:
-                fh = open(self.args.log_file)
+                fh = open(self.args.log_file, mode = 'r', encoding = 'utf-8')
             self.summer = summarizer.Summarizer(fh, **kwargs)
         else:
             self.summer = summarizer.Summarizer(sys.stdin, **kwargs)
diff --git a/tools/crunchstat-summary/crunchstat_summary/dygraphs.py b/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
index 1314e9df3..6df440a14 100644
--- a/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
+++ b/tools/crunchstat-summary/crunchstat_summary/dygraphs.py
@@ -18,7 +18,7 @@ class DygraphsChart(crunchstat_summary.webchart.WebChart):
             'data': self._collate_data(tasks, stat),
             'options': {
                 'connectSeparatedPoints': True,
-                'labels': ['elapsed']+[uuid for uuid, _ in tasks.iteritems()],
+                'labels': ['elapsed']+[uuid for uuid, _ in tasks.items()],
                 'title': '{}: {} {}'.format(label, stat[0], stat[1]),
             },
         }
@@ -26,7 +26,7 @@ class DygraphsChart(crunchstat_summary.webchart.WebChart):
     def _collate_data(self, tasks, stat):
         data = []
         nulls = []
-        for uuid, task in tasks.iteritems():
+        for uuid, task in tasks.items():
             for pt in task.series[stat]:
                 data.append([pt[0].total_seconds()] + nulls + [pt[1]])
             nulls.append(None)
diff --git a/tools/crunchstat-summary/crunchstat_summary/reader.py b/tools/crunchstat-summary/crunchstat_summary/reader.py
index 311c006c0..f96b7ee92 100644
--- a/tools/crunchstat-summary/crunchstat_summary/reader.py
+++ b/tools/crunchstat-summary/crunchstat_summary/reader.py
@@ -2,11 +2,11 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-from __future__ import print_function
+
 
 import arvados
 import itertools
-import Queue
+import queue
 import threading
 
 from crunchstat_summary import logger
@@ -87,19 +87,21 @@ class LiveLogReader(object):
             self._queue.put(self.EOF)
 
     def __iter__(self):
-        self._queue = Queue.Queue()
+        self._queue = queue.Queue()
         self._thread = threading.Thread(target=self._get_all_pages)
         self._thread.daemon = True
         self._thread.start()
         return self
 
-    def next(self):
+    def __next__(self):
         line = self._queue.get()
         if line is self.EOF:
             self._thread.join()
             raise StopIteration
         return line
 
+    next = __next__ # for Python 2
+
     def __enter__(self):
         return self
 
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index b2f6f1bb7..16e073011 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-from __future__ import print_function
+
 
 import arvados
 import collections
@@ -209,7 +209,7 @@ class Summarizer(object):
                     stats['user+sys'] = stats.get('user', 0) + stats.get('sys', 0)
                 if 'tx' in stats or 'rx' in stats:
                     stats['tx+rx'] = stats.get('tx', 0) + stats.get('rx', 0)
-                for stat, val in stats.iteritems():
+                for stat, val in stats.items():
                     if group == 'interval':
                         if stat == 'seconds':
                             this_interval_s = val
@@ -236,9 +236,9 @@ class Summarizer(object):
 
         self.job_tot = collections.defaultdict(
             functools.partial(collections.defaultdict, int))
-        for task_id, task_stat in self.task_stats.iteritems():
-            for category, stat_last in task_stat.iteritems():
-                for stat, val in stat_last.iteritems():
+        for task_id, task_stat in self.task_stats.items():
+            for category, stat_last in task_stat.items():
+                for stat, val in stat_last.items():
                     if stat in ['cpus', 'cache', 'swap', 'rss']:
                         # meaningless stats like 16 cpu cores x 5 tasks = 80
                         continue
@@ -273,8 +273,8 @@ class Summarizer(object):
 
     def _text_report_gen(self):
         yield "\t".join(['category', 'metric', 'task_max', 'task_max_rate', 'job_total'])
-        for category, stat_max in sorted(self.stats_max.iteritems()):
-            for stat, val in sorted(stat_max.iteritems()):
+        for category, stat_max in sorted(self.stats_max.items()):
+            for stat, val in sorted(stat_max.items()):
                 if stat.endswith('__rate'):
                     continue
                 max_rate = self._format(stat_max.get(stat+'__rate', '-'))
@@ -539,7 +539,7 @@ class MultiSummarizer(object):
 
     def run(self):
         threads = []
-        for child in self.children.itervalues():
+        for child in self.children.values():
             self.throttle.acquire()
             t = threading.Thread(target=self.run_and_release, args=(child.run, ))
             t.daemon = True
@@ -551,7 +551,7 @@ class MultiSummarizer(object):
     def text_report(self):
         txt = ''
         d = self._descendants()
-        for child in d.itervalues():
+        for child in d.values():
             if len(d) > 1:
                 txt += '### Summary for {} ({})\n'.format(
                     child.label, child.process['uuid'])
@@ -566,7 +566,7 @@ class MultiSummarizer(object):
         MultiSummarizers) are omitted.
         """
         d = collections.OrderedDict()
-        for key, child in self.children.iteritems():
+        for key, child in self.children.items():
             if isinstance(child, Summarizer):
                 d[key] = child
             if isinstance(child, MultiSummarizer):
@@ -574,7 +574,7 @@ class MultiSummarizer(object):
         return d
 
     def html_report(self):
-        return WEBCHART_CLASS(self.label, self._descendants().itervalues()).html()
+        return WEBCHART_CLASS(self.label, iter(self._descendants().values())).html()
 
 
 class JobTreeSummarizer(MultiSummarizer):
@@ -588,7 +588,7 @@ class JobTreeSummarizer(MultiSummarizer):
             preloaded = {}
             for j in arv.jobs().index(
                     limit=len(job['components']),
-                    filters=[['uuid','in',job['components'].values()]]).execute()['items']:
+                    filters=[['uuid','in',list(job['components'].values())]]).execute()['items']:
                 preloaded[j['uuid']] = j
             for cname in sorted(job['components'].keys()):
                 child_uuid = job['components'][cname]
@@ -605,7 +605,7 @@ class JobTreeSummarizer(MultiSummarizer):
 class PipelineSummarizer(MultiSummarizer):
     def __init__(self, instance, **kwargs):
         children = collections.OrderedDict()
-        for cname, component in instance['components'].iteritems():
+        for cname, component in instance['components'].items():
             if 'job' not in component:
                 logger.warning(
                     "%s: skipping component with no job assigned", cname)
@@ -663,7 +663,7 @@ class ContainerTreeSummarizer(MultiSummarizer):
                         cr['name'] = cr.get('name') or cr['uuid']
                         todo.append(cr)
         sorted_children = collections.OrderedDict()
-        for uuid in sorted(children.keys(), key=lambda uuid: children[uuid].sort_key):
+        for uuid in sorted(list(children.keys()), key=lambda uuid: children[uuid].sort_key):
             sorted_children[uuid] = children[uuid]
         super(ContainerTreeSummarizer, self).__init__(
             children=sorted_children,
diff --git a/tools/crunchstat-summary/crunchstat_summary/webchart.py b/tools/crunchstat-summary/crunchstat_summary/webchart.py
index 9d18883ce..6931937b0 100644
--- a/tools/crunchstat-summary/crunchstat_summary/webchart.py
+++ b/tools/crunchstat-summary/crunchstat_summary/webchart.py
@@ -33,7 +33,7 @@ class WebChart(object):
     def js(self):
         return 'var chartdata = {};\n{}'.format(
             json.dumps(self.sections()),
-            '\n'.join([pkg_resources.resource_string('crunchstat_summary', jsa) for jsa in self.JSASSETS]))
+            '\n'.join([pkg_resources.resource_string('crunchstat_summary', jsa).decode('utf-8') for jsa in self.JSASSETS]))
 
     def sections(self):
         return [
@@ -41,7 +41,7 @@ class WebChart(object):
                 'label': s.long_label(),
                 'charts': [
                     self.chartdata(s.label, s.tasks, stat)
-                    for stat in (('cpu', 'user+sys__rate'),
+                    for stat in (('cp', 'user+sys__rate'),
                                  ('mem', 'rss'),
                                  ('net:eth0', 'tx+rx__rate'),
                                  ('net:keep0', 'tx+rx__rate'))],
diff --git a/tools/crunchstat-summary/tests/test_examples.py b/tools/crunchstat-summary/tests/test_examples.py
index af92becd8..dd57cda2d 100644
--- a/tools/crunchstat-summary/tests/test_examples.py
+++ b/tools/crunchstat-summary/tests/test_examples.py
@@ -8,20 +8,23 @@ import crunchstat_summary.command
 import difflib
 import glob
 import gzip
+from io import open
 import mock
 import os
 import unittest
 
+from crunchstat_summary.command import UTF8Decode
+
 TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 class ReportDiff(unittest.TestCase):
     def diff_known_report(self, logfile, cmd):
         expectfile = logfile+'.report'
-        expect = open(expectfile).readlines()
+        expect = open(expectfile, encoding='utf-8').readlines()
         self.diff_report(cmd, expect, expectfile=expectfile)
 
-    def diff_report(self, cmd, expect, expectfile=None):
+    def diff_report(self, cmd, expect, expectfile='(expected)'):
         got = [x+"\n" for x in cmd.report().strip("\n").split("\n")]
         self.assertEqual(got, expect, "\n"+"".join(difflib.context_diff(
             expect, got, fromfile=expectfile, tofile="(generated)")))
@@ -54,7 +57,7 @@ class HTMLFromFile(ReportDiff):
 
 class SummarizeEdgeCases(unittest.TestCase):
     def test_error_messages(self):
-        logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'))
+        logfile = open(os.path.join(TESTS_DIR, 'crunchstat_error_messages.txt'), encoding='utf-8')
         s = crunchstat_summary.summarizer.Summarizer(logfile)
         s.run()
 
@@ -89,9 +92,9 @@ class SummarizeContainer(ReportDiff):
             'container.json', 'crunchstat.txt', 'arv-mount.txt']
         def _open(n):
             if n == "crunchstat.txt":
-                return gzip.open(self.logfile)
+                return UTF8Decode(gzip.open(self.logfile))
             elif n == "arv-mount.txt":
-                return gzip.open(self.arvmountlog)
+                return UTF8Decode(gzip.open(self.arvmountlog))
         mock_cr().open.side_effect = _open
         args = crunchstat_summary.command.ArgumentParser().parse_args(
             ['--job', self.fake_request['uuid']])
@@ -114,7 +117,7 @@ class SummarizeJob(ReportDiff):
     def test_job_report(self, mock_api, mock_cr):
         mock_api().jobs().get().execute.return_value = self.fake_job
         mock_cr().__iter__.return_value = ['fake-logfile.txt']
-        mock_cr().open.return_value = gzip.open(self.logfile)
+        mock_cr().open.return_value = UTF8Decode(gzip.open(self.logfile))
         args = crunchstat_summary.command.ArgumentParser().parse_args(
             ['--job', self.fake_job_uuid])
         cmd = crunchstat_summary.command.Command(args)
@@ -175,14 +178,14 @@ class SummarizePipeline(ReportDiff):
         mock_api().pipeline_instances().get().execute. \
             return_value = self.fake_instance
         mock_cr().__iter__.return_value = ['fake-logfile.txt']
-        mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
+        mock_cr().open.side_effect = [UTF8Decode(gzip.open(logfile)) for _ in range(3)]
         args = crunchstat_summary.command.ArgumentParser().parse_args(
             ['--pipeline-instance', self.fake_instance['uuid']])
         cmd = crunchstat_summary.command.Command(args)
         cmd.run()
 
         job_report = [
-            line for line in open(logfile+'.report').readlines()
+            line for line in open(logfile+'.report', encoding='utf-8').readlines()
             if not line.startswith('#!! ')]
         expect = (
             ['### Summary for foo (zzzzz-8i9sb-000000000000000)\n'] +
@@ -251,14 +254,14 @@ class SummarizeACRJob(ReportDiff):
         mock_api().jobs().index().execute.return_value = self.fake_jobs_index
         mock_api().jobs().get().execute.return_value = self.fake_job
         mock_cr().__iter__.return_value = ['fake-logfile.txt']
-        mock_cr().open.side_effect = [gzip.open(logfile) for _ in range(3)]
+        mock_cr().open.side_effect = [UTF8Decode(gzip.open(logfile)) for _ in range(3)]
         args = crunchstat_summary.command.ArgumentParser().parse_args(
             ['--job', self.fake_job['uuid']])
         cmd = crunchstat_summary.command.Command(args)
         cmd.run()
 
         job_report = [
-            line for line in open(logfile+'.report').readlines()
+            line for line in open(logfile+'.report', encoding='utf-8').readlines()
             if not line.startswith('#!! ')]
         expect = (
             ['### Summary for zzzzz-8i9sb-i3e77t9z5y8j9cc (partial) (zzzzz-8i9sb-i3e77t9z5y8j9cc)\n',

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list