[ARVADOS] updated: 47c349ca374356163d15e59deb917a4d22f88206
git at public.curoverse.com
git at public.curoverse.com
Mon Nov 3 14:25:52 EST 2014
Summary of changes:
sdk/python/arvados/collection.py | 13 ++++++-------
sdk/python/arvados/config.py | 2 +-
sdk/python/arvados/keep.py | 2 +-
sdk/python/arvados/stream.py | 2 +-
sdk/python/arvados/util.py | 26 ++++++++++++++++++++++----
sdk/python/tests/run_test_server.py | 8 ++++----
sdk/python/tests/test_collections.py | 3 ++-
services/fuse/arvados_fuse/__init__.py | 3 ++-
8 files changed, 39 insertions(+), 20 deletions(-)
via 47c349ca374356163d15e59deb917a4d22f88206 (commit)
via ab3b9556865e694a4c33ed623cf2e55fc410f10f (commit)
via 3b408ec4031c84549383db1ca99364e4c53a29fd (commit)
via 6c7e0599e34d037c0e280b7622969488fb8f8b2e (commit)
from ce2362b3b62ccb8fe2e5db5a2b87e9a6212ed86b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 47c349ca374356163d15e59deb917a4d22f88206
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 3 13:33:06 2014 -0500
3706: Add back "collections.get => 404" to "fallback to Keep" test.
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index c0ef118..34fd603 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -733,7 +733,8 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
# crunch-job needs this to read manifests that have only ever
# been written to Keep.
client = self.api_client_mock(200)
- with tutil.mock_responses(self.DEFAULT_MANIFEST, 404, 200):
+ self.mock_get_collection(client, 404, None)
+ with tutil.mock_responses(self.DEFAULT_MANIFEST, 200):
reader = arvados.CollectionReader(self.DEFAULT_DATA_HASH,
api_client=client)
self.assertEqual(self.DEFAULT_MANIFEST, reader.manifest_text())
commit ab3b9556865e694a4c33ed623cf2e55fc410f10f
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 3 09:35:59 2014 -0500
3706: Prefer "is None" and "is not None" to "== None" and "!= None".
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 3e92964..7467813 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -51,7 +51,7 @@ def normalize_stream(s, stream):
fout = f.replace(' ', '\\040')
for segment in stream[f]:
segmentoffset = blocks[segment[arvados.LOCATOR]] + segment[arvados.OFFSET]
- if current_span == None:
+ if current_span is None:
current_span = [segmentoffset, segmentoffset + segment[arvados.SEGMENTSIZE]]
else:
if segmentoffset == current_span[1]:
@@ -60,7 +60,7 @@ def normalize_stream(s, stream):
stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
current_span = [segmentoffset, segmentoffset + segment[arvados.SEGMENTSIZE]]
- if current_span != None:
+ if current_span is not None:
stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
if not stream[f]:
@@ -185,7 +185,7 @@ class CollectionReader(CollectionBase):
error_via_keep = self._populate_from_keep()
if not self._manifest_text:
error_via_api = self._populate_from_api_server()
- if error_via_api != None and not should_try_keep:
+ if error_via_api is not None and not should_try_keep:
raise error_via_api
if (not self._manifest_text and
not error_via_keep and
@@ -417,7 +417,7 @@ class CollectionWriter(CollectionBase):
return self._current_file_name
def finish_current_file(self):
- if self._current_file_name == None:
+ if self._current_file_name is None:
if self._current_file_pos == self._current_stream_length:
return
raise errors.AssertionError(
diff --git a/sdk/python/arvados/config.py b/sdk/python/arvados/config.py
index 66595e9..a0c3cc6 100644
--- a/sdk/python/arvados/config.py
+++ b/sdk/python/arvados/config.py
@@ -7,7 +7,7 @@ import os
import re
_settings = None
-if os.environ.get('HOME') != None:
+if os.environ.get('HOME') is not None:
default_config_file = os.environ['HOME'] + '/.config/arvados/settings.conf'
else:
default_config_file = ''
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 0144a10..46bd1cb 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -165,7 +165,7 @@ class KeepBlockCache(object):
self.ready.set()
def size(self):
- if self.content == None:
+ if self.content is None:
return 0
else:
return len(self.content)
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index c9a0613..f5975b9 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -225,7 +225,7 @@ class StreamReader(object):
# parse stream
for tok in tokens:
if debug: print 'tok', tok
- if self._stream_name == None:
+ if self._stream_name is None:
self._stream_name = tok.replace('\\040', ' ')
continue
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 5014055..a474a94 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -24,7 +24,7 @@ def clear_tmpdir(path=None):
Ensure the given directory (or TASK_TMPDIR if none given)
exists and is empty.
"""
- if path == None:
+ if path is None:
path = arvados.current_task().tmpdir
if os.path.exists(path):
p = subprocess.Popen(['rm', '-rf', path])
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index bd3419c..82c6424 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -43,7 +43,7 @@ def find_server_pid(PID_PATH, wait=10):
try:
with open(PID_PATH, 'r') as f:
server_pid = int(f.read())
- good_pid = (os.kill(server_pid, 0) == None)
+ good_pid = (os.kill(server_pid, 0) is None)
except IOError:
good_pid = False
except OSError:
@@ -62,7 +62,7 @@ def kill_server_pid(PID_PATH, wait=10):
with open(PID_PATH, 'r') as f:
server_pid = int(f.read())
while now <= timeout:
- os.kill(server_pid, signal.SIGTERM) == None
+ os.kill(server_pid, signal.SIGTERM)
os.getpgid(server_pid) # throw OSError if no such pid
now = time.time()
time.sleep(0.1)
@@ -82,7 +82,7 @@ def run(websockets=False, reuse_server=False):
test_pid = find_server_pid(pid_file, 0)
- if test_pid == None or not reuse_server:
+ if test_pid is None or not reuse_server:
# do not try to run both server variants at once
stop()
@@ -314,7 +314,7 @@ if __name__ == "__main__":
if args.action == 'start':
run(websockets=args.websockets, reuse_server=args.reuse)
- if args.auth != None:
+ if args.auth is not None:
authorize_with(args.auth)
print("export ARVADOS_API_HOST={}".format(arvados.config.settings()["ARVADOS_API_HOST"]))
print("export ARVADOS_API_TOKEN={}".format(arvados.config.settings()["ARVADOS_API_TOKEN"]))
commit 3b408ec4031c84549383db1ca99364e4c53a29fd
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 3 09:33:16 2014 -0500
3706: Mutating normalize() does not return self.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 07a5c56..d0f2643 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -314,7 +314,8 @@ class CollectionDirectory(Directory):
self.clear()
collection = arvados.CollectionReader(
self.collection_object["manifest_text"], self.api,
- self.api.localkeep(), num_retries=self.num_retries).normalize()
+ self.api.localkeep(), num_retries=self.num_retries)
+ collection.normalize()
for s in collection.all_streams():
cwd = self
for part in s.name().split('/'):
commit 6c7e0599e34d037c0e280b7622969488fb8f8b2e
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Nov 3 09:32:16 2014 -0500
3706: Process directory trees in sorted() order, instead of the order
returned by os.listdir().
This causes write_directory_tree() to produce a manifest that is
normalized to the greatest extent possible for the given
max_manifest_depth.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 82ef156..3e92964 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -348,9 +348,8 @@ class CollectionWriter(CollectionBase):
def _work_trees(self):
path, stream_name, max_manifest_depth = self._queued_trees[0]
- make_dirents = (util.listdir_recursive if (max_manifest_depth == 0)
- else os.listdir)
- d = make_dirents(path)
+ d = util.listdir_recursive(
+ path, max_depth = (None if max_manifest_depth == 0 else 0))
if d:
self._queue_dirents(stream_name, d)
else:
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 2532ee2..5014055 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -308,13 +308,31 @@ def stream_extract(stream, path, files=[], decompress=True):
lockfile.close()
return path
-def listdir_recursive(dirname, base=None):
+def listdir_recursive(dirname, base=None, max_depth=None):
+ """listdir_recursive(dirname, base, max_depth)
+
+ Return a list of file and directory names found under dirname.
+
+ If base is not None, prepend "{base}/" to each returned name.
+
+ If max_depth is None, descend into directories and return only the
+ names of files found in the directory tree.
+
+ If max_depth is a non-negative integer, stop descending into
+ directories at the given depth, and at that point return directory
+ names instead.
+
+ If max_depth==0 (and base is None) this is equivalent to
+ sorted(os.listdir(dirname)).
+ """
allfiles = []
for ent in sorted(os.listdir(dirname)):
ent_path = os.path.join(dirname, ent)
ent_base = os.path.join(base, ent) if base else ent
- if os.path.isdir(ent_path):
- allfiles += listdir_recursive(ent_path, ent_base)
+ if os.path.isdir(ent_path) and max_depth != 0:
+ allfiles += listdir_recursive(
+ ent_path, base=ent_base,
+ max_depth=(max_depth-1 if max_depth else None))
else:
allfiles += [ent_base]
return allfiles
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list