[ARVADOS] updated: a603d0f7b685a93b88ea29336dd2b512e63b4508
Git user
git at public.curoverse.com
Mon Apr 10 21:59:51 EDT 2017
Summary of changes:
sdk/python/arvados/arvfile.py | 5 +++--
sdk/python/arvados/cache.py | 2 +-
sdk/python/arvados/collection.py | 17 +++++++++--------
sdk/python/arvados/commands/arv_copy.py | 3 ++-
sdk/python/arvados/commands/put.py | 9 +++++----
sdk/python/arvados/stream.py | 5 +++--
sdk/python/tests/run_test_server.py | 1 +
sdk/python/tests/test_cache.py | 11 +++++------
sdk/python/tests/test_events.py | 2 +-
9 files changed, 30 insertions(+), 25 deletions(-)
via a603d0f7b685a93b88ea29336dd2b512e63b4508 (commit)
via 9b976cfbcb62729a672315107ac4868e0dc2f815 (commit)
via 83f3b04387aff7f033b99a6421e4d38478f09c17 (commit)
via 62be7d11e2b54d06e4d0c7c6ae12656bfe00637f (commit)
from 9e3fd2fa437e2f3dab220a96fbe22ec5b250421d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a603d0f7b685a93b88ea29336dd2b512e63b4508
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 10 20:23:04 2017 -0400
11308: Remove superfluous parens.
diff --git a/sdk/python/tests/test_events.py b/sdk/python/tests/test_events.py
index 84b586f..5e1e006 100644
--- a/sdk/python/tests/test_events.py
+++ b/sdk/python/tests/test_events.py
@@ -35,7 +35,7 @@ class WebsocketTest(run_test_server.TestCaseWithServers):
if self.ws:
self.ws.close()
except Exception as e:
- print(("Error in teardown: ", e))
+ print("Error in teardown: ", e)
super(WebsocketTest, self).tearDown()
run_test_server.reset()
commit 9b976cfbcb62729a672315107ac4868e0dc2f815
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 10 20:22:42 2017 -0400
11308: Avoid Python2-inefficient list() operations.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 180b120..c52c772 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -1,6 +1,7 @@
from __future__ import absolute_import
from __future__ import division
from future import standard_library
+from future.utils import listitems, listvalues
standard_library.install_aliases()
from builtins import range
from builtins import object
@@ -590,7 +591,7 @@ class _BlockManager(object):
# A WRITABLE block with its owner.closed() implies that it's
# size is <= KEEP_BLOCK_SIZE/2.
try:
- small_blocks = [b for b in list(self._bufferblocks.values()) if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
+ small_blocks = [b for b in listvalues(self._bufferblocks) if b.state() == _BufferBlock.WRITABLE and b.owner.closed()]
except AttributeError:
# Writable blocks without owner shouldn't exist.
raise UnownedBlockError()
@@ -703,7 +704,7 @@ class _BlockManager(object):
self.repack_small_blocks(force=True, sync=True)
with self.lock:
- items = list(self._bufferblocks.items())
+ items = listitems(self._bufferblocks)
for k,v in items:
if v.state() != _BufferBlock.COMMITTED and v.owner:
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 5b3b8d5..5008450 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1,4 +1,5 @@
from __future__ import absolute_import
+from future.utils import listitems, listvalues, viewkeys
from builtins import str
from past.builtins import basestring
from builtins import object
@@ -427,7 +428,7 @@ class ResumableCollectionWriter(CollectionWriter):
return writer
def check_dependencies(self):
- for path, orig_stat in list(self._dependencies.items()):
+ for path, orig_stat in listitems(self._dependencies):
if not S_ISREG(orig_stat[ST_MODE]):
raise errors.StaleWriterStateError("{} not file".format(path))
try:
@@ -681,7 +682,7 @@ class RichCollectionBase(CollectionBase):
if value == self._committed:
return
if value:
- for k,v in list(self._items.items()):
+ for k,v in listitems(self._items):
v.set_committed(True)
self._committed = True
else:
@@ -692,7 +693,7 @@ class RichCollectionBase(CollectionBase):
@synchronized
def __iter__(self):
"""Iterate over names of files and collections contained in this collection."""
- return iter(list(self._items.keys()))
+ return iter(viewkeys(self._items))
@synchronized
def __getitem__(self, k):
@@ -724,17 +725,17 @@ class RichCollectionBase(CollectionBase):
@synchronized
def keys(self):
"""Get a list of names of files and collections directly contained in this collection."""
- return list(self._items.keys())
+ return viewkeys(self._items)
@synchronized
def values(self):
"""Get a list of files and collection objects directly contained in this collection."""
- return list(self._items.values())
+ return listvalues(self._items)
@synchronized
def items(self):
"""Get a list of (name, object) tuples directly contained in this collection."""
- return list(self._items.items())
+ return listitems(self._items)
def exists(self, path):
"""Test if there is a file or collection at `path`."""
@@ -767,7 +768,7 @@ class RichCollectionBase(CollectionBase):
item.remove(pathcomponents[1])
def _clonefrom(self, source):
- for k,v in list(source.items()):
+ for k,v in listitems(source):
self._items[k] = v.clone(self, k)
def clone(self):
@@ -1125,7 +1126,7 @@ class RichCollectionBase(CollectionBase):
@synchronized
def flush(self):
"""Flush bufferblocks to Keep."""
- for e in list(self.values()):
+ for e in listvalues(self):
e.flush()
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index c0d05b3..8242e10 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -16,6 +16,7 @@
from __future__ import division
from future import standard_library
+from future.utils import listvalues
standard_library.install_aliases()
from past.builtins import basestring
from builtins import object
@@ -155,7 +156,7 @@ def main():
abort("cannot copy object {} of type {}".format(args.object_uuid, t))
# Clean up any outstanding temp git repositories.
- for d in list(local_repo_dir.values()):
+ for d in listvalues(local_repo_dir):
shutil.rmtree(d, ignore_errors=True)
# If no exception was thrown and the response does not have an
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 400d57d..08ed576 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -1,4 +1,5 @@
from __future__ import division
+from future.utils import listitems, listvalues
from builtins import str
from builtins import object
import argparse
@@ -508,7 +509,7 @@ class ArvPutUploadJob(object):
Recursively get the total size of the collection
"""
size = 0
- for item in list(collection.values()):
+ for item in listvalues(collection):
if isinstance(item, arvados.collection.Collection) or isinstance(item, arvados.collection.Subcollection):
size += self._collection_size(item)
else:
@@ -700,7 +701,7 @@ class ArvPutUploadJob(object):
def collection_file_paths(self, col, path_prefix='.'):
"""Return a list of file paths by recursively go through the entire collection `col`"""
file_paths = []
- for name, item in list(col.items()):
+ for name, item in listitems(col):
if isinstance(item, arvados.arvfile.ArvadosFile):
file_paths.append(os.path.join(path_prefix, name))
elif isinstance(item, arvados.collection.Subcollection):
@@ -777,7 +778,7 @@ class ArvPutUploadJob(object):
locators.append(loc)
return locators
elif isinstance(item, arvados.collection.Collection):
- l = [self._datablocks_on_item(x) for x in list(item.values())]
+ l = [self._datablocks_on_item(x) for x in listvalues(item)]
# Fast list flattener method taken from:
# http://stackoverflow.com/questions/952914/making-a-flat-list-out-of-list-of-lists-in-python
return [loc for sublist in l for loc in sublist]
@@ -981,7 +982,7 @@ def main(arguments=None, stdout=sys.stdout, stderr=sys.stderr):
if not output.endswith('\n'):
stdout.write('\n')
- for sigcode, orig_handler in list(orig_signal_handlers.items()):
+ for sigcode, orig_handler in listitems(orig_signal_handlers):
signal.signal(sigcode, orig_handler)
if status != 0:
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index b68685e..86ae8a5 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -1,5 +1,6 @@
from __future__ import print_function
from __future__ import absolute_import
+from future.utils import listvalues
from builtins import object
import collections
import hashlib
@@ -63,7 +64,7 @@ class StreamReader(object):
return self._files
def all_files(self):
- return list(self._files.values())
+ return listvalues(self._files)
def size(self):
n = self._data_locators[-1]
@@ -98,5 +99,5 @@ class StreamReader(object):
manifest_text.extend([d.locator for d in self._data_locators])
manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040'))
for seg in f.segments])
- for f in list(self._files.values())])
+ for f in listvalues(self._files)])
return ' '.join(manifest_text) + '\n'
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 5aff52b..6d9ac06 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -1,5 +1,6 @@
from __future__ import print_function
from __future__ import division
+from future.utils import viewkeys
from builtins import str
from builtins import range
import argparse
commit 83f3b04387aff7f033b99a6421e4d38478f09c17
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 10 20:22:03 2017 -0400
11308: Fix futurize misinterpretation.
diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index f59d92f..0c5061b 100644
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -40,7 +40,7 @@ class SafeHTTPCache(object):
try:
with open(filename, 'rb') as f:
return f.read()
- except IOError as OSError:
+ except (IOError, OSError):
return None
def set(self, url, content):
commit 62be7d11e2b54d06e4d0c7c6ae12656bfe00637f
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Apr 10 20:21:19 2017 -0400
11308: Put back exception handling
diff --git a/sdk/python/tests/test_cache.py b/sdk/python/tests/test_cache.py
index 6adab7b..02a8817 100644
--- a/sdk/python/tests/test_cache.py
+++ b/sdk/python/tests/test_cache.py
@@ -34,17 +34,16 @@ class CacheTestThread(threading.Thread):
for x in range(16):
try:
data_in = _random(128)
- data_in = bytes(hashlib.md5(data_in).hexdigest()) + bytes("\n") + bytes(data_in)
+ data_in = hashlib.md5(data_in).hexdigest().encode() + b"\n" + data_in
c.set(url, data_in)
data_out = c.get(url)
digest, _, content = data_out.partition("\n")
if digest != bytes(hashlib.md5(content).hexdigest()):
self.ok = False
- finally:
- pass
- #except Exception as err:
- # self.ok = False
- # print("cache failed: {}: {}".format(type(err), err), file=sys.stderr)
+ except Exception as err:
+ self.ok = False
+ print("cache failed: {}: {}".format(type(err), err), file=sys.stderr)
+ raise
class CacheTest(unittest.TestCase):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list