[ARVADOS] updated: c691e44f9183176581dd8da6417af16772baf510
git at public.curoverse.com
git at public.curoverse.com
Mon Mar 2 15:55:13 EST 2015
Summary of changes:
sdk/python/arvados/__init__.py | 12 ++++-----
sdk/python/arvados/_ranges.py | 11 +++------
sdk/python/arvados/api.py | 6 ++---
sdk/python/arvados/arvfile.py | 14 +++++------
sdk/python/arvados/collection.py | 52 ++++++++++++++++++++++-----------------
sdk/python/arvados/errors.py | 2 +-
sdk/python/arvados/keep.py | 5 +++-
sdk/python/tests/test_arvfile.py | 3 ---
sdk/python/tests/test_sdk.py | 2 +-
services/fuse/tests/test_mount.py | 1 +
10 files changed, 55 insertions(+), 53 deletions(-)
via c691e44f9183176581dd8da6417af16772baf510 (commit)
from 63b03a39adfd78961c5bbb6a3a2d02ccd8c92e4d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c691e44f9183176581dd8da6417af16772baf510
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Mar 2 15:51:55 2015 -0500
4823: More fixes and cleanups.
* Renamed SynchronizedCollectionBase to RichCollectionBase
* Renamed arvapi parameter of one_task_per_input_file to api_client
* KeepLocator.stripped() returns bare hash if self.size is None
* Permit closing an ArvadosFileWriter more than once
* Fix various docstrings
* Strive to follow PEP 8 spacing guidelines
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index ed7e6a9..1df6470 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -83,15 +83,15 @@ class JobTask(object):
class job_setup:
@staticmethod
- def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, arvapi=None):
+ def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, api_client=None):
if if_sequence != current_task()['sequence']:
return
- if not arvapi:
- arvapi = api('v1')
+ if not api_client:
+ api_client = api('v1')
job_input = current_job()['script_parameters']['input']
- cr = CollectionReader(job_input, api_client=arvapi)
+ cr = CollectionReader(job_input, api_client=api_client)
cr.normalize()
for s in cr.all_streams():
for f in s.all_files():
@@ -107,9 +107,9 @@ class job_setup:
'input':task_input
}
}
- arvapi.job_tasks().create(body=new_task_attrs).execute()
+ api_client.job_tasks().create(body=new_task_attrs).execute()
if and_end_task:
- arvapi.job_tasks().update(uuid=current_task()['uuid'],
+ api_client.job_tasks().update(uuid=current_task()['uuid'],
body={'success':True}
).execute()
exit(0)
diff --git a/sdk/python/arvados/_ranges.py b/sdk/python/arvados/_ranges.py
index 6f22bad..ba92f93 100644
--- a/sdk/python/arvados/_ranges.py
+++ b/sdk/python/arvados/_ranges.py
@@ -66,11 +66,12 @@ class LocatorAndRange(object):
return "LocatorAndRange(%r, %r, %r, %r)" % (self.locator, self.block_size, self.segment_offset, self.segment_size)
def locators_and_ranges(data_locators, range_start, range_size):
- """Get blocks that are covered by the range and return list of LocatorAndRange
- objects.
+ """Get blocks that are covered by a range.
+
+ Returns a list of LocatorAndRange objects.
:data_locators:
- list of Range objects, assumes that blocks are in order and contigous
+ list of Range objects, assumes that blocks are in order and contiguous
:range_start:
start of range
@@ -82,8 +83,6 @@ def locators_and_ranges(data_locators, range_start, range_size):
if range_size == 0:
return []
resp = []
- range_start = range_start
- range_size = range_size
range_end = range_start + range_size
i = first_block(data_locators, range_start, range_size)
@@ -144,8 +143,6 @@ def replace_range(data_locators, new_range_start, new_range_size, new_locator, n
if new_range_size == 0:
return
- new_range_start = new_range_start
- new_range_size = new_range_size
new_range_end = new_range_start + new_range_size
if len(data_locators) == 0:
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 830be96..a227e43 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -158,10 +158,6 @@ def api_from_config(version=None, apiconfig=None, **kwargs):
A string naming the version of the Arvados REST API to use (for
example, 'v1').
- :cache:
- Use a cache (~/.cache/arvados/discovery) for the discovery
- document.
-
:apiconfig:
If provided, this should be a dict-like object (must support the get()
method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
@@ -169,6 +165,8 @@ def api_from_config(version=None, apiconfig=None, **kwargs):
arvados.config (which gets these parameters from the environment by
default.)
+ Other keyword arguments such as `cache` will be passed along `api()`
+
"""
# Load from user configuration or environment
if apiconfig is None:
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index f9f91cf..c820309 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -342,6 +342,7 @@ class NoopLock(object):
def release(self):
pass
+
def must_be_writable(orig_func):
@functools.wraps(orig_func)
def must_be_writable_wrapper(self, *args, **kwargs):
@@ -797,7 +798,6 @@ class ArvadosFile(object):
self._repack_writes()
self.parent._my_block_manager().commit_bufferblock(self._current_bblock)
-
@must_be_writable
@synchronized
def add_segment(self, blocks, pos, size):
@@ -826,16 +826,14 @@ class ArvadosFile(object):
else:
return 0
-
@synchronized
def manifest_text(self, stream_name=".", portable_locators=False, normalize=False):
buf = ""
- item = self
filestream = []
- for segment in item.segments:
+ for segment in self.segments:
loc = segment.locator
if loc.startswith("bufferblock"):
- loc = item._bufferblocks[loc].calculate_locator()
+ loc = self._bufferblocks[loc].calculate_locator()
if portable_locators:
loc = KeepLocator(loc).stripped()
filestream.append(LocatorAndRange(loc, locator_block_size(loc),
@@ -919,7 +917,7 @@ class ArvadosFileWriter(ArvadosFileReader):
def flush(self):
self.arvadosfile.flush()
- @_FileLikeObjectBase._before_close
def close(self):
- self.flush()
- super(ArvadosFileWriter, self).close()
+ if not self.closed:
+ self.flush()
+ super(ArvadosFileWriter, self).close()
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index b0cdd92..3d48652 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -470,13 +470,14 @@ class ResumableCollectionWriter(CollectionWriter):
"resumable writer can't accept unsourced data")
return super(ResumableCollectionWriter, self).write(data)
+
ADD = "add"
DEL = "del"
MOD = "mod"
FILE = "file"
COLLECTION = "collection"
-class SynchronizedCollectionBase(CollectionBase):
+class RichCollectionBase(CollectionBase):
"""Base class for Collections and Subcollections.
Implements the majority of functionality relating to accessing items in the
@@ -550,7 +551,7 @@ class SynchronizedCollectionBase(CollectionBase):
self._items[pathcomponents[0]] = item
self._modified = True
self.notify(ADD, self, pathcomponents[0], item)
- if isinstance(item, SynchronizedCollectionBase):
+ if isinstance(item, RichCollectionBase):
return item.find_or_create(pathcomponents[1], create_type)
else:
raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
@@ -573,7 +574,7 @@ class SynchronizedCollectionBase(CollectionBase):
if len(pathcomponents) == 1:
return item
else:
- if isinstance(item, SynchronizedCollectionBase):
+ if isinstance(item, RichCollectionBase):
if pathcomponents[1]:
return item.find(pathcomponents[1])
else:
@@ -702,7 +703,7 @@ class SynchronizedCollectionBase(CollectionBase):
def exists(self, path):
"""Test if there is a file or collection at `path`."""
- return self.find(path) != None
+ return self.find(path) is not None
@must_be_writable
@synchronized
@@ -721,7 +722,7 @@ class SynchronizedCollectionBase(CollectionBase):
if item is None:
raise IOError((errno.ENOENT, "File not found"))
if len(pathcomponents) == 1:
- if isinstance(self._items[pathcomponents[0]], SynchronizedCollectionBase) and len(self._items[pathcomponents[0]]) > 0 and not recursive:
+ if isinstance(self._items[pathcomponents[0]], RichCollectionBase) and len(self._items[pathcomponents[0]]) > 0 and not recursive:
raise IOError((errno.ENOTEMPTY, "Subcollection not empty"))
deleteditem = self._items[pathcomponents[0]]
del self._items[pathcomponents[0]]
@@ -771,7 +772,6 @@ class SynchronizedCollectionBase(CollectionBase):
else:
self.notify(ADD, self, target_name, dup)
-
@must_be_writable
@synchronized
def copy(self, source, target_path, source_collection=None, overwrite=False):
@@ -816,7 +816,7 @@ class SynchronizedCollectionBase(CollectionBase):
target_dir = self.find_or_create("/".join(targetcomponents[0:-1]), COLLECTION)
- if target_name in target_dir and isinstance(self[target_name], SynchronizedCollectionBase) and sourcecomponents:
+ if target_name in target_dir and isinstance(self[target_name], RichCollectionBase) and sourcecomponents:
target_dir = target_dir[target_name]
target_name = sourcecomponents[-1]
@@ -842,13 +842,12 @@ class SynchronizedCollectionBase(CollectionBase):
"""
if self.modified() or self._manifest_text is None or normalize:
- item = self
stream = {}
buf = []
- sorted_keys = sorted(item.keys())
- for filename in [s for s in sorted_keys if isinstance(item[s], ArvadosFile)]:
+ sorted_keys = sorted(self.keys())
+ for filename in [s for s in sorted_keys if isinstance(self[s], ArvadosFile)]:
# Create a stream per file `k`
- arvfile = item[filename]
+ arvfile = self[filename]
filestream = []
for segment in arvfile.segments():
loc = segment.locator
@@ -861,8 +860,8 @@ class SynchronizedCollectionBase(CollectionBase):
stream[filename] = filestream
if stream:
buf.append(" ".join(normalize_stream(stream_name, stream)) + "\n")
- for dirname in [s for s in sorted_keys if isinstance(item[s], SynchronizedCollectionBase)]:
- buf.append(item[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip))
+ for dirname in [s for s in sorted_keys if isinstance(self[s], RichCollectionBase)]:
+ buf.append(self[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip))
return "".join(buf)
else:
if strip:
@@ -949,7 +948,7 @@ class SynchronizedCollectionBase(CollectionBase):
def __eq__(self, other):
if other is self:
return True
- if not isinstance(other, SynchronizedCollectionBase):
+ if not isinstance(other, RichCollectionBase):
return False
if len(self._items) != len(other):
return False
@@ -964,7 +963,7 @@ class SynchronizedCollectionBase(CollectionBase):
return not self.__eq__(other)
-class Collection(SynchronizedCollectionBase):
+class Collection(RichCollectionBase):
"""Represents the root of an Arvados Collection.
This class is threadsafe. The root collection object, all subcollections
@@ -1188,7 +1187,16 @@ class Collection(SynchronizedCollectionBase):
@synchronized
def manifest_locator(self):
- """Get the manifest locator. May be None."""
+ """Get the manifest locator, if any.
+
+ The manifest locator will be set when the collection is loaded from an
+ API server record or the portable data hash of a manifest.
+
+ The manifest locator will be None if the collection is newly created or
+ was created directly from manifest text. The method `save_new()` will
+ assign a manifest locator.
+
+ """
return self._manifest_locator
@synchronized
@@ -1214,21 +1222,21 @@ class Collection(SynchronizedCollectionBase):
return self._api_response
def find_or_create(self, path, create_type):
- """See `SynchronizedCollectionBase.find_or_create`"""
+ """See `RichCollectionBase.find_or_create`"""
if path == ".":
return self
else:
return super(Collection, self).find_or_create(path[2:] if path.startswith("./") else path, create_type)
def find(self, path):
- """See `SynchronizedCollectionBase.find`"""
+ """See `RichCollectionBase.find`"""
if path == ".":
return self
else:
return super(Collection, self).find(path[2:] if path.startswith("./") else path)
def remove(self, path, recursive=False):
- """See `SynchronizedCollectionBase.remove`"""
+ """See `RichCollectionBase.remove`"""
if path == ".":
raise errors.ArgumentError("Cannot remove '.'")
else:
@@ -1241,8 +1249,8 @@ class Collection(SynchronizedCollectionBase):
"""Save collection to an existing collection record.
Commit pending buffer blocks to Keep, merge with remote record (if
- update=True), write the manifest to Keep, and update the collection
- record.
+ merge=True, the default), write the manifest to Keep, and update the
+ collection record.
Will raise AssertionError if not associated with a collection record on
the API server. If you want to save a manifest to Keep only, see
@@ -1401,7 +1409,7 @@ class Collection(SynchronizedCollectionBase):
self.set_unmodified()
-class Subcollection(SynchronizedCollectionBase):
+class Subcollection(RichCollectionBase):
"""This is a subdirectory within a collection that doesn't have its own API
server record.
diff --git a/sdk/python/arvados/errors.py b/sdk/python/arvados/errors.py
index f6681d6..ab4609f 100644
--- a/sdk/python/arvados/errors.py
+++ b/sdk/python/arvados/errors.py
@@ -17,7 +17,7 @@ class ApiError(apiclient_errors.HttpError):
class KeepRequestError(Exception):
"""Base class for errors accessing Keep services."""
def __init__(self, message='', request_errors=(), label=""):
- """KeepRequestError(message='', request_errors=())
+ """KeepRequestError(message='', request_errors=(), label="")
:message:
A human-readable message describing what Keep operation
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 8d0a89d..62c6709 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -59,7 +59,10 @@ class KeepLocator(object):
if s is not None)
def stripped(self):
- return "%s+%i" % (self.md5sum, self.size)
+ if self.size is not None:
+ return "%s+%i" % (self.md5sum, self.size)
+ else:
+ return self.md5sum
def _make_hex_prop(name, length):
# Build and return a new property with the given name that
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index f50c446..825465c 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -405,9 +405,6 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
def _my_block_manager(self):
return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
- def sync_mode(self):
- return SYNC_READONLY
-
def make_count_reader(self, nocache=False):
stream = []
diff --git a/sdk/python/tests/test_sdk.py b/sdk/python/tests/test_sdk.py
index 8c56180..20f0d3d 100644
--- a/sdk/python/tests/test_sdk.py
+++ b/sdk/python/tests/test_sdk.py
@@ -39,5 +39,5 @@ class TestSDK(unittest.TestCase):
# Because one_task_per_input_file normalizes this collection,
# it should now create only one job task and not three.
- arvados.job_setup.one_task_per_input_file(and_end_task=False, arvapi=mock_api)
+ arvados.job_setup.one_task_per_input_file(and_end_task=False, api_client=mock_api)
mock_api.job_tasks().create().execute.assert_called_once_with()
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index f747f93..764a099 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -1,4 +1,5 @@
import arvados
+import arvados.safeapi
import arvados_fuse as fuse
import glob
import json
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list