[ARVADOS] updated: 93d38f3cb3d7874ed3723409357d8dd5e8d618a5
git at public.curoverse.com
git at public.curoverse.com
Fri Feb 20 15:56:02 EST 2015
Summary of changes:
sdk/python/arvados/__init__.py | 2 +-
sdk/python/arvados/_normalize_stream.py | 53 +++
sdk/python/arvados/{ranges.py => _ranges.py} | 8 +-
sdk/python/arvados/{api.py => apisetup.py} | 66 ++-
sdk/python/arvados/arvfile.py | 115 +++--
sdk/python/arvados/collection.py | 638 ++++++++-------------------
sdk/python/arvados/safeapi.py | 12 +-
sdk/python/arvados/stream.py | 56 +--
sdk/python/tests/run_test_server.py | 2 +-
sdk/python/tests/test_arvfile.py | 119 ++---
sdk/python/tests/test_collections.py | 181 +++-----
sdk/python/tests/test_sdk.py | 3 +-
sdk/python/tests/test_stream.py | 3 +-
13 files changed, 501 insertions(+), 757 deletions(-)
create mode 100644 sdk/python/arvados/_normalize_stream.py
rename sdk/python/arvados/{ranges.py => _ranges.py} (99%)
rename sdk/python/arvados/{api.py => apisetup.py} (74%)
via 93d38f3cb3d7874ed3723409357d8dd5e8d618a5 (commit)
via 62e3f155b14b30fde1323a367351c70b23f312b8 (commit)
from f39118dacff5614e1102bb18f6dc850aecf2186e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 93d38f3cb3d7874ed3723409357d8dd5e8d618a5
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 20 15:58:14 2015 -0500
4823: Fix tests broken by prior refactoring. Renamed 'api.py' to 'apisetup.py'
so it wouldn't be awkwardly shadowed by the 'api' method.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index e4d148e..91fe460 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -18,7 +18,7 @@ import fcntl
import time
import threading
-from api import *
+from apisetup import api, http_cache
from collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
from keep import *
from stream import *
diff --git a/sdk/python/arvados/_normalize_stream.py b/sdk/python/arvados/_normalize_stream.py
index 4423639..400a38e 100644
--- a/sdk/python/arvados/_normalize_stream.py
+++ b/sdk/python/arvados/_normalize_stream.py
@@ -9,6 +9,7 @@ def normalize_stream(stream_name, stream):
"""
+ stream_name = stream_name.replace(' ', '\\040')
stream_tokens = [stream_name]
sortedfiles = list(stream.keys())
sortedfiles.sort()
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/apisetup.py
similarity index 94%
rename from sdk/python/arvados/api.py
rename to sdk/python/arvados/apisetup.py
index cf1c29b..830be96 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/apisetup.py
@@ -76,14 +76,22 @@ def http_cache(data_type):
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
- Arguments:
- * version: A string naming the version of the Arvados API to use (for
+ :version:
+ A string naming the version of the Arvados API to use (for
example, 'v1').
- * cache: Use a cache (~/.cache/arvados/discovery) for the discovery
+
+ :cache:
+ Use a cache (~/.cache/arvados/discovery) for the discovery
document.
- * host: The Arvados API server host (and optional :port) to connect to.
- * token: The authentication token to send with each API call.
- * insecure: If True, ignore SSL certificate validation errors.
+
+ :host:
+ The Arvados API server host (and optional :port) to connect to.
+
+ :token:
+ The authentication token to send with each API call.
+
+ :insecure:
+ If True, ignore SSL certificate validation errors.
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index b5eb6e6..d9d9cd2 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -2,7 +2,7 @@ import functools
import os
import zlib
import bz2
-from ._ranges import locators_and_ranges, replace_range
+from ._ranges import locators_and_ranges, replace_range, Range
from arvados.retry import retry_method
import config
import hashlib
@@ -81,7 +81,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
def decompressed_name(self):
return re.sub('\.(bz2|gz)$', '', self.name)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def seek(self, pos, whence=os.SEEK_CUR):
if whence == os.SEEK_CUR:
pos += self._filepos
@@ -92,7 +92,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
def tell(self):
return self._filepos
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readall(self, size=2**20, num_retries=None):
while True:
@@ -101,7 +101,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
break
yield data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readline(self, size=float('inf'), num_retries=None):
cache_pos, cache_data = self._readline_cache
@@ -125,7 +125,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
self._readline_cache = (self.tell(), data[nextline_index:])
return data[:nextline_index]
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def decompress(self, decompress, size, num_retries=None):
for segment in self.readall(size, num_retries):
@@ -133,7 +133,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
if data:
yield data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readall_decompressed(self, size=2**20, num_retries=None):
self.seek(0)
@@ -148,7 +148,7 @@ class ArvadosFileReaderBase(_FileLikeObjectBase):
else:
return self.readall(size, num_retries=num_retries)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readlines(self, sizehint=float('inf'), num_retries=None):
data = []
@@ -183,7 +183,7 @@ class StreamFileReader(ArvadosFileReaderBase):
n = self.segments[-1]
return n.range_start + n.range_size
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def read(self, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at the current file position"""
@@ -201,7 +201,7 @@ class StreamFileReader(ArvadosFileReaderBase):
self._filepos += len(data)
return data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readfrom(self, start, size, num_retries=None):
"""Read up to 'size' bytes from the stream, starting at 'start'"""
@@ -300,9 +300,9 @@ class _BufferBlock(object):
(self._state == _BufferBlock.PENDING and nextstate == _BufferBlock.COMMITTED)):
self._state = nextstate
if self._state == _BufferBlock.COMMITTED:
- bufferblock._locator = loc
- bufferblock.buffer_view = None
- bufferblock.buffer_block = None
+ self._locator = loc
+ self.buffer_view = None
+ self.buffer_block = None
else:
raise AssertionError("Invalid state change from %s to %s" % (self.state, state))
@@ -821,6 +821,7 @@ class ArvadosFile(object):
@synchronized
def manifest_text(self, stream_name=".", portable_locators=False, normalize=False):
+ buf = ""
item = self
filestream = []
for segment in item.segments:
@@ -852,7 +853,10 @@ class ArvadosFileReader(ArvadosFileReaderBase):
def size(self):
return self.arvadosfile.size()
- @FileLikeObjectBase._before_close
+ def stream_name(self):
+ return self.arvadosfile.parent.stream_name()
+
+ @_FileLikeObjectBase._before_close
@retry_method
def read(self, size, num_retries=None):
"""Read up to `size` bytes from the stream, starting at the current file position."""
@@ -860,7 +864,7 @@ class ArvadosFileReader(ArvadosFileReaderBase):
self._filepos += len(data)
return data
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def readfrom(self, offset, size, num_retries=None):
"""Read up to `size` bytes from the stream, starting at the current file position."""
@@ -881,7 +885,7 @@ class ArvadosFileWriter(ArvadosFileReader):
def __init__(self, arvadosfile, name, mode, num_retries=None):
super(ArvadosFileWriter, self).__init__(arvadosfile, name, mode, num_retries=num_retries)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def write(self, data, num_retries=None):
if self.mode[0] == "a":
@@ -890,7 +894,7 @@ class ArvadosFileWriter(ArvadosFileReader):
self.arvadosfile.writeto(self._filepos, data, num_retries)
self._filepos += len(data)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
@retry_method
def writelines(self, seq, num_retries=None):
for s in seq:
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 82b9745..08b9deb 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -8,10 +8,10 @@ import time
from collections import deque
from stat import *
-from .arvfile import split, FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, _BlockManager, synchronized, must_be_writable, SYNC_READONLY, SYNC_EXPLICIT, NoopLock
+from .arvfile import split, _FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, _BlockManager, synchronized, must_be_writable, SYNC_READONLY, SYNC_EXPLICIT, NoopLock
from keep import *
-from .stream import StreamReader, normalize_stream, locator_block_size
-from .ranges import Range, LocatorAndRange
+from .stream import StreamReader, normalize_stream
+from ._ranges import Range, LocatorAndRange
from .safeapi import ThreadSafeApiCache
import config
import errors
@@ -54,7 +54,7 @@ class CollectionBase(object):
return ''.join(clean)
-class _WriterFile(FileLikeObjectBase):
+class _WriterFile(_FileLikeObjectBase):
def __init__(self, coll_writer, name):
super(_WriterFile, self).__init__(name, 'wb')
self.dest = coll_writer
@@ -63,16 +63,16 @@ class _WriterFile(FileLikeObjectBase):
super(_WriterFile, self).close()
self.dest.finish_current_file()
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def write(self, data):
self.dest.write(data)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def writelines(self, seq):
for data in seq:
self.write(data)
- @FileLikeObjectBase._before_close
+ @_FileLikeObjectBase._before_close
def flush(self):
self.dest.flush_data()
@@ -494,9 +494,6 @@ class SynchronizedCollectionBase(CollectionBase):
def _my_block_manager(self):
raise NotImplementedError()
- def _populate(self):
- raise NotImplementedError()
-
def sync_mode(self):
raise NotImplementedError()
@@ -506,6 +503,9 @@ class SynchronizedCollectionBase(CollectionBase):
def notify(self, event, collection, name, item):
raise NotImplementedError()
+ def stream_name(self):
+ raise NotImplementedError()
+
@must_be_writable
@synchronized
def find_or_create(self, path, create_type):
@@ -554,7 +554,7 @@ class SynchronizedCollectionBase(CollectionBase):
if isinstance(item, SynchronizedCollectionBase):
return item.find_or_create("/".join(pathcomponents), create_type)
else:
- raise errors.ArgumentError("Interior path components must be subcollection")
+ raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
else:
return self
@@ -580,7 +580,7 @@ class SynchronizedCollectionBase(CollectionBase):
if isinstance(item, SynchronizedCollectionBase):
return item.find("/".join(pathcomponents))
else:
- raise errors.ArgumentError("Interior path components must be subcollection")
+ raise IOError((errno.ENOTDIR, "Interior path components must be subcollection"))
else:
return self
@@ -593,7 +593,7 @@ class SynchronizedCollectionBase(CollectionBase):
"""
return self.find_or_create(path, COLLECTION)
- def open(self, path, mode):
+ def open(self, path, mode="r"):
"""Open a file-like object for access.
:path:
@@ -613,7 +613,7 @@ class SynchronizedCollectionBase(CollectionBase):
"""
mode = mode.replace("b", "")
if len(mode) == 0 or mode[0] not in ("r", "w", "a"):
- raise ArgumentError("Bad mode '%s'" % mode)
+ raise errors.ArgumentError("Bad mode '%s'" % mode)
create = (mode != "r")
if create and self.sync_mode() == SYNC_READONLY:
@@ -632,10 +632,12 @@ class SynchronizedCollectionBase(CollectionBase):
if mode[0] == "w":
arvfile.truncate(0)
+ name = os.path.basename(path)
+
if mode == "r":
- return ArvadosFileReader(arvfile, path, mode, num_retries=self.num_retries)
+ return ArvadosFileReader(arvfile, name, mode, num_retries=self.num_retries)
else:
- return ArvadosFileWriter(arvfile, path, mode, num_retries=self.num_retries)
+ return ArvadosFileWriter(arvfile, name, mode, num_retries=self.num_retries)
@synchronized
def modified(self):
@@ -832,10 +834,10 @@ class SynchronizedCollectionBase(CollectionBase):
"""
- portable_locators = strip
if self.modified() or self._manifest_text is None or normalize:
item = self
stream = {}
+ buf = ""
sorted_keys = sorted(item.keys())
for filename in [s for s in sorted_keys if isinstance(item[s], ArvadosFile)]:
# Create a stream per file `k`
@@ -845,19 +847,19 @@ class SynchronizedCollectionBase(CollectionBase):
loc = segment.locator
if arvfile.parent._my_block_manager().is_bufferblock(loc):
loc = arvfile.parent._my_block_manager().get_bufferblock(loc).locator()
- if portable_locators:
+ if strip:
loc = KeepLocator(loc).stripped()
- filestream.append(LocatorAndRange(loc, locator_block_size(loc),
+ filestream.append(LocatorAndRange(loc, KeepLocator(loc).size,
segment.segment_offset, segment.range_size))
stream[filename] = filestream
if stream:
buf += ' '.join(normalize_stream(stream_name, stream))
buf += "\n"
for dirname in [s for s in sorted_keys if isinstance(item[s], SynchronizedCollectionBase)]:
- buf += item[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), portable_locators=portable_locators)
+ buf += item[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), strip=strip)
return buf
else:
- if portable_locators:
+ if strip:
return self.stripped_manifest()
else:
return self._manifest_text
@@ -870,7 +872,7 @@ class SynchronizedCollectionBase(CollectionBase):
"""
changes = []
if holding_collection is None:
- holding_collection = Collection(api_client=self._my_api(), keep_client=self._my_keep(), sync=SYNC_EXPLICIT)
+ holding_collection = Collection(api_client=self._my_api(), keep_client=self._my_keep())
for k in self:
if k not in end_collection:
changes.append((DEL, os.path.join(prefix, k), self[k].clone(holding_collection)))
@@ -1029,14 +1031,13 @@ class Collection(SynchronizedCollectionBase):
else:
self._config = config.settings()
- self.num_retries = num_retries if num_retries is not None else 2
+ self.num_retries = num_retries if num_retries is not None else 0
self._manifest_locator = None
self._manifest_text = None
self._api_response = None
self._sync = SYNC_EXPLICIT
- if not self.lock:
- self.lock = threading.RLock()
+ self.lock = threading.RLock()
self.callbacks = []
self.events = None
@@ -1057,6 +1058,9 @@ class Collection(SynchronizedCollectionBase):
def root_collection(self):
return self
+ def stream_name(self):
+ return "."
+
def sync_mode(self):
return self._sync
@@ -1071,8 +1075,8 @@ class Collection(SynchronizedCollectionBase):
if self._manifest_locator is None:
raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
- other = import_manifest(response["manifest_text"])
- baseline = import_manifest(self._manifest_text)
+ other = CollectionReader(response["manifest_text"])
+ baseline = CollectionReader(self._manifest_text)
self.apply(baseline.diff(other))
@synchronized
@@ -1088,7 +1092,7 @@ class Collection(SynchronizedCollectionBase):
if self._api_client is None:
self._my_api()
else:
- self._keep_client = KeepClient(api=self._api_client)
+ self._keep_client = KeepClient(api_client=self._api_client)
return self._keep_client
@synchronized
@@ -1156,7 +1160,7 @@ class Collection(SynchronizedCollectionBase):
error_via_keep))
# populate
self._baseline_manifest = self._manifest_text
- import_manifest(self._manifest_text, self)
+ self._import_manifest(self._manifest_text)
def _has_collection_uuid(self):
@@ -1173,14 +1177,17 @@ class Collection(SynchronizedCollectionBase):
self._block_manager.stop_threads()
@synchronized
- def clone(self, new_parent=None, new_sync=SYNC_READONLY, new_config=None):
+ def clone(self, new_parent=None, readonly=False, new_config=None):
if new_config is None:
new_config = self._config
- newcollection = Collection(parent=new_parent, apiconfig=new_config, sync=SYNC_EXPLICIT)
- if new_sync == SYNC_READONLY:
- newcollection.lock = NoopLock()
+ if readonly:
+ newcollection = CollectionReader(parent=new_parent, apiconfig=new_config)
+ else:
+ newcollection = Collection(parent=new_parent, apiconfig=new_config)
+
+ newcollection._sync = None
self._cloneinto(newcollection)
- newcollection._sync = new_sync
+ newcollection._sync = SYNC_READONLY if readonly else SYNC_EXPLICIT
return newcollection
@synchronized
@@ -1363,6 +1370,7 @@ class Subcollection(SynchronizedCollectionBase):
def __init__(self, parent):
super(Subcollection, self).__init__(parent)
self.lock = self.root_collection().lock
+ self._manifest_text = None
def root_collection(self):
return self.parent.root_collection()
@@ -1379,12 +1387,15 @@ class Subcollection(SynchronizedCollectionBase):
def _my_block_manager(self):
return self.root_collection()._my_block_manager()
- def _populate(self):
- self.root_collection()._populate()
-
def notify(self, event, collection, name, item):
return self.root_collection().notify(event, collection, name, item)
+ def stream_name(self):
+ for k, v in self.parent.items():
+ if v is self:
+ return os.path.join(self.parent.stream_name(), k)
+ return '.'
+
@synchronized
def clone(self, new_parent):
c = Subcollection(new_parent)
@@ -1403,16 +1414,20 @@ class CollectionReader(Collection):
if not args and not kwargs.get("manifest_locator_or_text"):
raise errors.ArgumentError("Must provide manifest locator or text to initialize ReadOnlyCollection")
+ super(CollectionReader, self).__init__(*args, **kwargs)
+
# Forego any locking since it should never change once initialized.
self.lock = NoopLock()
-
- super(ReadOnlyCollection, self).__init__(*args, **kwargs)
-
self._sync = SYNC_READONLY
- self._streams = [sline.split()
- for sline in self._manifest_text.split("\n")
- if sline]
+ # Backwards compatability with old CollectionReader
+ # all_streams() and all_files()
+ if self._manifest_text:
+ self._streams = [sline.split()
+ for sline in self._manifest_text.split("\n")
+ if sline]
+ else:
+ self._streams = []
def normalize(self):
# Rearrange streams
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index 5c5c872..baada91 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -1,5 +1,5 @@
import threading
-import api
+import apisetup
import keep
import config
import copy
@@ -21,7 +21,7 @@ class ThreadSafeApiCache(object):
def localapi(self):
if 'api' not in self.local.__dict__:
- self.local.api = api.api_from_config('v1', apiconfig=self.apiconfig)
+ self.local.api = apisetup.api_from_config('v1', apiconfig=self.apiconfig)
return self.local.api
def __getattr__(self, name):
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index a6e0863..85c0320 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -6,7 +6,7 @@ import threading
import functools
import copy
-from ._ranges import locators_and_ranges
+from ._ranges import locators_and_ranges, Range
from .arvfile import StreamFileReader
from arvados.retry import retry_method
from keep import *
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 18011af..b9502f0 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -25,7 +25,7 @@ if __name__ == '__main__' and os.path.exists(
# Add the Python SDK source to the library path.
sys.path.insert(1, os.path.dirname(MY_DIRNAME))
-import arvados.api
+import arvados
import arvados.config
ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index e5670e3..b6e8426 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -9,8 +9,9 @@ import unittest
import hashlib
import arvados
-from arvados import Range, KeepLocator
-from arvados.collection import import_manifest, export_manifest, ReadOnlyCollection, WritableCollection
+from arvados._ranges import Range
+from arvados.keep import KeepLocator
+from arvados.collection import Collection, CollectionReader
from arvados.arvfile import ArvadosFile, ArvadosFileReader, SYNC_READONLY, SYNC_EXPLICIT
import arvados_testutil as tutil
@@ -61,7 +62,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
@@ -83,7 +84,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
@@ -110,7 +111,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_append(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
writer = c.open("count.txt", "a+")
self.assertEqual(writer.read(20), "0123456789")
writer.seek(0, os.SEEK_SET)
@@ -122,11 +123,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("world")
self.assertEqual(writer.read(20), "0123456789helloworld")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.manifest_text())
def test_write_at_beginning(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
@@ -134,11 +135,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("foo")
self.assertEqual(writer.size(), 10)
self.assertEqual("foo3456789", writer.readfrom(0, 13))
- self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", export_manifest(c))
+ self.assertEqual(". acbd18db4cc2f85cedef654fccc4a4d8+3 781e5e245d69b566979b86e28d23f2c7+10 0:3:count.txt 6:7:count.txt\n", c.manifest_text())
def test_write_in_middle(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
@@ -146,11 +147,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("foo")
self.assertEqual(writer.size(), 10)
self.assertEqual("012foo6789", writer.readfrom(0, 13))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:count.txt 10:3:count.txt 6:4:count.txt\n", c.manifest_text())
def test_write_at_end(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
@@ -158,11 +159,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("foo")
self.assertEqual(writer.size(), 10)
self.assertEqual("0123456foo", writer.readfrom(0, 13))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:7:count.txt 10:3:count.txt\n", c.manifest_text())
def test_write_across_segment_boundary(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012345678901234", writer.readfrom(0, 15))
@@ -170,11 +171,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("foobar")
self.assertEqual(writer.size(), 20)
self.assertEqual("0123456foobar34", writer.readfrom(0, 15))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 3858f62230ac3c915f300c664312c63f+6 0:7:count.txt 10:6:count.txt 3:7:count.txt\n", c.manifest_text())
def test_write_across_several_segments(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012301230123", writer.readfrom(0, 15))
@@ -182,7 +183,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.write("abcdefg")
self.assertEqual(writer.size(), 12)
self.assertEqual("01abcdefg123", writer.readfrom(0, 15))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 7ac66c0f148de9519b8bd264312c4d64+7 0:2:count.txt 10:7:count.txt 1:3:count.txt\n", c.manifest_text())
def test_write_large(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
@@ -190,7 +191,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"})
- with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
@@ -206,7 +207,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_rewrite_on_empty_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
@@ -215,11 +216,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.arvadosfile._repack_writes()
self.assertEqual(writer.size(), 10)
self.assertEqual("0123456789", writer.readfrom(0, 20))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n", c.manifest_text())
def test_rewrite_append_existing_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
@@ -228,11 +229,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.arvadosfile._repack_writes()
self.assertEqual(writer.size(), 20)
self.assertEqual("0123456789abcdefghij", writer.readfrom(0, 20))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n", c.manifest_text())
def test_rewrite_over_existing_file(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
@@ -241,7 +242,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
writer.arvadosfile._repack_writes()
self.assertEqual(writer.size(), 15)
self.assertEqual("01234abcdefghij", writer.readfrom(0, 20))
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:5:count.txt 10:10:count.txt\n", c.manifest_text())
def test_write_large_rewrite(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
@@ -249,7 +250,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"})
- with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
@@ -271,7 +272,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
- with WritableCollection(api_client=api, keep_client=keep) as c:
+ with Collection(api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
@@ -292,7 +293,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
- with WritableCollection(api_client=api, keep_client=keep) as c:
+ with Collection(api_client=api, keep_client=keep) as c:
self.assertIsNone(c.api_response())
writer = c.open("foo/bar/count.txt", "w+")
writer.write("01234567")
@@ -305,7 +306,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
@@ -319,12 +320,12 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertEqual(False, c.modified())
def test_file_not_found(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open("nocount.txt", "r")
def test_cannot_open_directory(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open(".", "r")
@@ -334,7 +335,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"})
- with WritableCollection(api_client=api, keep_client=keep) as c:
+ with Collection(api_client=api, keep_client=keep) as c:
w1 = c.open("count1.txt", "w")
w2 = c.open("count2.txt", "w")
w1.write("01234567")
@@ -410,21 +411,21 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
def test_prefetch(self):
keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
- with WritableCollection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
+ with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
r = c.open("count.txt", "r")
self.assertEqual("0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
def test__eq__from_manifest(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
def test__eq__from_writes(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with WritableCollection() as c2:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with Collection() as c2:
with c2.open("count1.txt", "w") as f:
f.write("0123456789")
@@ -432,8 +433,8 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
def test__ne__(self):
- with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with WritableCollection() as c2:
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with Collection() as c2:
with c2.open("count1.txt", "w") as f:
f.write("1234567890")
@@ -454,7 +455,7 @@ class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
n += k.size
except ValueError:
pass
- col = ReadOnlyCollection(keep_client=self.keep_client())
+ col = Collection(keep_client=self.keep_client())
col._my_block_manager().prefetch_enabled = False
af = ArvadosFile(col,
stream=stream,
@@ -487,7 +488,7 @@ class ArvadosFileReadlinesTestCase(ArvadosFileReadTestCase):
class BlockManagerTest(unittest.TestCase):
def test_bufferblock_append(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- blockmanager = arvados.arvfile.BlockManager(keep)
+ blockmanager = arvados.arvfile._BlockManager(keep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
@@ -501,20 +502,20 @@ class BlockManagerTest(unittest.TestCase):
self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
- bufferblock.set_state(arvados.arvfile.BufferBlock.PENDING)
+ bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
with self.assertRaises(arvados.errors.AssertionError):
bufferblock.append("bar")
def test_bufferblock_dup(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- blockmanager = arvados.arvfile.BlockManager(keep)
+ blockmanager = arvados.arvfile._BlockManager(keep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
self.assertEqual(bufferblock.size(), 3)
self.assertEqual(bufferblock.buffer_view[0:3], "foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
- bufferblock.set_state(arvados.arvfile.BufferBlock.PENDING)
+ bufferblock.set_state(arvados.arvfile._BufferBlock.PENDING)
bufferblock2 = blockmanager.dup_block(bufferblock, None)
self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
@@ -531,7 +532,7 @@ class BlockManagerTest(unittest.TestCase):
def test_bufferblock_get(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- blockmanager = arvados.arvfile.BlockManager(keep)
+ blockmanager = arvados.arvfile._BlockManager(keep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
@@ -540,22 +541,22 @@ class BlockManagerTest(unittest.TestCase):
def test_bufferblock_commit(self):
mockkeep = mock.MagicMock()
- blockmanager = arvados.arvfile.BlockManager(mockkeep)
+ blockmanager = arvados.arvfile._BlockManager(mockkeep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
blockmanager.commit_all()
self.assertTrue(mockkeep.put.called)
- self.assertEqual(bufferblock.state(), arvados.arvfile.BufferBlock.COMMITTED)
+ self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.COMMITTED)
self.assertIsNone(bufferblock.buffer_view)
def test_bufferblock_commit_with_error(self):
mockkeep = mock.MagicMock()
mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
- blockmanager = arvados.arvfile.BlockManager(mockkeep)
+ blockmanager = arvados.arvfile._BlockManager(mockkeep)
bufferblock = blockmanager.alloc_bufferblock()
bufferblock.append("foo")
with self.assertRaises(arvados.errors.KeepWriteError) as err:
blockmanager.commit_all()
self.assertEquals(str(err.exception), "Error writing some blocks: acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
- self.assertEqual(bufferblock.state(), arvados.arvfile.BufferBlock.PENDING)
+ self.assertEqual(bufferblock.state(), arvados.arvfile._BufferBlock.PENDING)
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index de06b16..aebfbd0 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -13,10 +13,10 @@ import tempfile
import unittest
import run_test_server
-import arvados_testutil as tutil
-from arvados.ranges import Range, LocatorAndRange
-from arvados.collection import import_manifest, export_manifest, ReadOnlyCollection, WritableCollection
+from arvados._ranges import Range, LocatorAndRange
+from arvados.collection import Collection, CollectionReader
from arvados.arvfile import SYNC_EXPLICIT
+import arvados_testutil as tutil
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
@@ -546,9 +546,8 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
def test_uuid_init_failure_raises_api_error(self):
client = self.api_client_mock(500)
- reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
with self.assertRaises(arvados.errors.ApiError):
- reader.manifest_text()
+ reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
def test_locator_init(self):
client = self.api_client_mock(200)
@@ -571,11 +570,10 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
def test_uuid_init_no_fallback_to_keep(self):
# Do not look up a collection UUID in Keep.
client = self.api_client_mock(404)
- reader = arvados.CollectionReader(self.DEFAULT_UUID,
- api_client=client)
with tutil.mock_get_responses(self.DEFAULT_MANIFEST, 200):
with self.assertRaises(arvados.errors.ApiError):
- reader.manifest_text()
+ reader = arvados.CollectionReader(self.DEFAULT_UUID,
+ api_client=client)
def test_try_keep_first_if_permission_hint(self):
# To verify that CollectionReader tries Keep first here, we
@@ -651,12 +649,6 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
cfile = reader.open('./foo')
self.check_open_file(cfile, '.', 'foo', 3)
- def test_open_collection_file_two_arguments(self):
- client = self.api_client_mock(200)
- reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- cfile = reader.open('.', 'foo')
- self.check_open_file(cfile, '.', 'foo', 3)
-
def test_open_deep_file(self):
coll_name = 'collection_with_files_in_subdir'
client = self.api_client_mock(200)
@@ -670,12 +662,12 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
def test_open_nonexistent_stream(self):
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- self.assertRaises(ValueError, reader.open, './nonexistent', 'foo')
+ self.assertRaises(IOError, reader.open, './nonexistent/foo')
def test_open_nonexistent_file(self):
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- self.assertRaises(ValueError, reader.open, '.', 'nonexistent')
+ self.assertRaises(IOError, reader.open, 'nonexistent')
@tutil.skip_sleep
@@ -816,75 +808,70 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
- def test_import_export_manifest(self):
- m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
-. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
-. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
-"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", export_manifest(import_manifest(m1)))
def test_init_manifest(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", export_manifest(ReadOnlyCollection(m1)))
+ self.assertEqual(m1, CollectionReader(m1).manifest_text(normalize=False))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", CollectionReader(m1).manifest_text(normalize=True))
def test_remove(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
self.assertIn("count1.txt", c)
c.remove("count1.txt")
self.assertNotIn("count1.txt", c)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.manifest_text())
def test_remove_in_subdir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_remove_empty_subdir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
c.remove("foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_remove_nonempty_subdir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
with self.assertRaises(IOError):
c.remove("foo")
c.remove("foo", recursive=True)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_copy_to_file_in_dir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", c.manifest_text())
def test_copy_file(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_copy_to_existing_dir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.copy("count1.txt", "foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_copy_to_new_dir(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", c.manifest_text())
def test_clone(self):
- c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
cl = c.clone()
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", cl.manifest_text())
def test_diff_del_add(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
('add', './count1.txt', c1["count1.txt"])])
@@ -896,8 +883,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_same(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [])
d = c1.diff(c2)
@@ -908,8 +895,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_mod(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
d = c1.diff(c2)
@@ -920,8 +907,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_add(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
d = c1.diff(c2)
@@ -932,8 +919,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_add_in_subcollection(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo', c2["foo"])])
d = c1.diff(c2)
@@ -944,8 +931,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_del_add_in_subcollection(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
@@ -959,8 +946,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff_mod_in_subcollection(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
- c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
d = c1.diff(c2)
@@ -971,8 +958,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_conflict_keep_local_change(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
('add', './count2.txt', c2["count2.txt"])])
@@ -984,8 +971,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), ". 95ebc3c7b3b9f1d2c40fec14415d3cb8+5 5348b82a029fd9e971a811ce1f71360b+43 0:5:count1.txt 5:10:count2.txt\n")
def test_conflict_mod(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
with c1.open("count1.txt", "w") as f:
@@ -997,8 +984,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.manifest_text()))
def test_conflict_add(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]),
('add', './count1.txt', c2["count1.txt"])])
@@ -1011,8 +998,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.manifest_text()))
def test_conflict_del(self):
- c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
c1.remove("count1.txt")
@@ -1023,14 +1010,14 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.manifest_text()))
def test_notify(self):
- c1 = WritableCollection()
+ c1 = Collection()
events = []
c1.subscribe(lambda event, collection, name, item: events.append((event, collection, name, item)))
f = c1.open("foo.txt", "w")
self.assertEqual(events[0], (arvados.collection.ADD, c1, "foo.txt", f.arvadosfile))
def test_open_w(self):
- c1 = WritableCollection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
+ c1 = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
self.assertEqual(c1["count1.txt"].size(), 10)
c1.open("count1.txt", "w").close()
self.assertEqual(c1["count1.txt"].size(), 0)
@@ -1040,8 +1027,12 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
MAIN_SERVER = {}
KEEP_SERVER = {}
- def test_create_and_save(self):
- c = arvados.collection.createWritableCollection("hello world")
+ def create_count_txt(self):
+ # Create an empty collection, save it to the API server, then write a
+ # file, but don't save it.
+
+ c = Collection()
+ c.save_new("CollectionCreateUpdateTest", ensure_unique_name=True)
self.assertEquals(c.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
self.assertEquals(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
@@ -1050,39 +1041,26 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
self.assertEquals(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
- c.save()
+ return c
+ def test_create_and_save(self):
+ c = self.create_count_txt()
+ c.save()
self.assertTrue(re.match(r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count.txt$",
c.manifest_text()))
def test_create_and_save_new(self):
- c = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
-
- with c.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c = self.create_count_txt()
c.save_new()
-
self.assertTrue(re.match(r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count.txt$",
c.manifest_text()))
def test_create_diff_apply(self):
- c1 = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c1.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c1 = self.create_count_txt()
c1.save()
- c2 = WritableCollection(c1._manifest_locator)
+ c2 = Collection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
f.write("abcdefg")
@@ -1094,9 +1072,9 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
self.assertEqual(c1.portable_data_hash(), c2.portable_data_hash())
def test_diff_apply_with_token(self):
- baseline = ReadOnlyCollection(". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:10:count.txt\n")
- c = WritableCollection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
- other = ReadOnlyCollection(". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:7:count.txt\n")
+ baseline = CollectionReader(". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:10:count.txt\n")
+ c = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
+ other = CollectionReader(". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:7:count.txt\n")
diff = baseline.diff(other)
self.assertEqual(diff, [('mod', u'./count.txt', c["count.txt"], other["count.txt"])])
@@ -1107,17 +1085,10 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
def test_create_and_update(self):
- c1 = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c1.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c1 = self.create_count_txt()
c1.save()
- c2 = arvados.collection.WritableCollection(c1._manifest_locator)
+ c2 = arvados.collection.Collection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
f.write("abcdefg")
@@ -1129,19 +1100,13 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
def test_create_and_update_with_conflict(self):
- c1 = arvados.collection.createWritableCollection("hello world")
- self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
- self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c1.open("count.txt", "w") as f:
- f.write("0123456789")
-
- self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
-
+ c1 = self.create_count_txt()
c1.save()
+
with c1.open("count.txt", "w") as f:
f.write("XYZ")
- c2 = arvados.collection.WritableCollection(c1._manifest_locator)
+ c2 = arvados.collection.Collection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
f.write("abcdefg")
diff --git a/sdk/python/tests/test_sdk.py b/sdk/python/tests/test_sdk.py
index 3436a07..d8b37d8 100644
--- a/sdk/python/tests/test_sdk.py
+++ b/sdk/python/tests/test_sdk.py
@@ -7,7 +7,7 @@ import arvados.collection
class TestSDK(unittest.TestCase):
- @mock.patch('arvados.api')
+ @mock.patch('arvados.apisetup.api_from_config')
@mock.patch('arvados.current_task')
@mock.patch('arvados.current_job')
def test_one_task_per_input_file_normalize(self, mock_job, mock_task, mock_api):
@@ -40,4 +40,3 @@ class TestSDK(unittest.TestCase):
# it should now create only one job task and not three.
arvados.job_setup.one_task_per_input_file(and_end_task=False)
mock_api('v1').job_tasks().create().execute.assert_called_once_with()
-
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 11ee694..5bf82cc 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -9,7 +9,8 @@ import unittest
import hashlib
import arvados
-from arvados import StreamReader, StreamFileReader, Range
+from arvados import StreamReader, StreamFileReader
+from arvados._ranges import Range
import arvados_testutil as tutil
import run_test_server
commit 62e3f155b14b30fde1323a367351c70b23f312b8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 20 13:29:10 2015 -0500
4823: Refactoring. ReadOnly Collection is now CollectionReader, replacing old
CollectionReader implementation (but maintains the same public API).
diff --git a/sdk/python/arvados/_normalize_stream.py b/sdk/python/arvados/_normalize_stream.py
new file mode 100644
index 0000000..4423639
--- /dev/null
+++ b/sdk/python/arvados/_normalize_stream.py
@@ -0,0 +1,52 @@
+def normalize_stream(stream_name, stream):
+ """Take manifest stream and return a list of tokens in normalized format.
+
+ :stream_name:
+ The name of the stream.
+
+ :stream:
+ A dict mapping each filename to a list of `_range.LocatorAndRange` objects.
+
+ """
+
+ stream_tokens = [stream_name]
+ sortedfiles = list(stream.keys())
+ sortedfiles.sort()
+
+ blocks = {}
+ streamoffset = 0L
+ # Go through each file and add each referenced block exactly once.
+ for streamfile in sortedfiles:
+ for segment in stream[streamfile]:
+ if segment.locator not in blocks:
+ stream_tokens.append(segment.locator)
+ blocks[segment.locator] = streamoffset
+ streamoffset += segment.block_size
+
+ # Add the empty block if the stream is otherwise empty.
+ if len(stream_tokens) == 1:
+ stream_tokens.append(config.EMPTY_BLOCK_LOCATOR)
+
+ for streamfile in sortedfiles:
+ # Add in file segments
+ current_span = None
+ fout = streamfile.replace(' ', '\\040')
+ for segment in stream[streamfile]:
+ # Collapse adjacent segments
+ streamoffset = blocks[segment.locator] + segment.segment_offset
+ if current_span is None:
+ current_span = [streamoffset, streamoffset + segment.segment_size]
+ else:
+ if streamoffset == current_span[1]:
+ current_span[1] += segment.segment_size
+ else:
+ stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
+ current_span = [streamoffset, streamoffset + segment.segment_size]
+
+ if current_span is not None:
+ stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
+
+ if not stream[streamfile]:
+ stream_tokens.append("0:0:{0}".format(fout))
+
+ return stream_tokens
diff --git a/sdk/python/arvados/ranges.py b/sdk/python/arvados/_ranges.py
similarity index 99%
rename from sdk/python/arvados/ranges.py
rename to sdk/python/arvados/_ranges.py
index 2a08b3b..1862d94 100644
--- a/sdk/python/arvados/ranges.py
+++ b/sdk/python/arvados/_ranges.py
@@ -66,7 +66,7 @@ class LocatorAndRange(object):
return "LocatorAndRange(\"%s\", %i, %i, %i)" % (self.locator, self.block_size, self.segment_offset, self.segment_size)
def locators_and_ranges(data_locators, range_start, range_size):
- '''Get blocks that are covered by the range and return list of LocatorAndRange
+ """Get blocks that are covered by the range and return list of LocatorAndRange
objects.
:data_locators:
@@ -78,7 +78,7 @@ def locators_and_ranges(data_locators, range_start, range_size):
:range_size:
size of range
- '''
+ """
if range_size == 0:
return []
resp = []
@@ -124,7 +124,7 @@ def locators_and_ranges(data_locators, range_start, range_size):
return resp
def replace_range(data_locators, new_range_start, new_range_size, new_locator, new_segment_offset):
- '''
+ """
Replace a file segment range with a new segment.
NOTE::
@@ -145,7 +145,7 @@ def replace_range(data_locators, new_range_start, new_range_size, new_locator, n
:new_segment_offset:
segment offset within the locator
- '''
+ """
if new_range_size == 0:
return
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index a44d330..cf1c29b 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -73,7 +73,7 @@ def http_cache(data_type):
path = None
return path
-def api(version=None, cache=True, host=None, token=None, insecure=False, apiconfig=None, **kwargs):
+def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
Arguments:
@@ -84,7 +84,6 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, apiconf
* host: The Arvados API server host (and optional :port) to connect to.
* token: The authentication token to send with each API call.
* insecure: If True, ignore SSL certificate validation errors.
- * apiconfig: If provided, this should be a dict containing with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and optionally ARVADOS_API_HOST_INSECURE
Additional keyword arguments will be passed directly to
`apiclient_discovery.build` if a new Resource object is created.
@@ -110,15 +109,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, apiconf
elif host and token:
pass
elif not host and not token:
- # Load from user configuration or environment
- if apiconfig is None:
- apiconfig = config.settings()
- for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
- if x not in apiconfig:
- raise ValueError("%s is not set. Aborting." % x)
- host = apiconfig.get('ARVADOS_API_HOST')
- token = apiconfig.get('ARVADOS_API_TOKEN')
- insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
+ return api_from_config(version=version, cache=cache, **kwargs)
else:
# Caller provided one but not the other
if not host:
@@ -150,3 +141,36 @@ def api(version=None, cache=True, host=None, token=None, insecure=False, apiconf
svc.api_token = token
kwargs['http'].cache = None
return svc
+
+def api_from_config(version=None, apiconfig=None, **kwargs):
+ """Return an apiclient Resources object enabling access to an Arvados server
+ instance.
+
+ :version:
+ A string naming the version of the Arvados REST API to use (for
+ example, 'v1').
+
+ :cache:
+ Use a cache (~/.cache/arvados/discovery) for the discovery
+ document.
+
+ :apiconfig:
+ If provided, this should be a dict-like object (must support the get()
+ method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
+ optionally ARVADOS_API_HOST_INSECURE. If not provided, use
+ arvados.config (which gets these parameters from the environment by
+ default.)
+
+ """
+ # Load from user configuration or environment
+ if apiconfig is None:
+ apiconfig = config.settings()
+
+ for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
+ if x not in apiconfig:
+ raise ValueError("%s is not set. Aborting." % x)
+ host = apiconfig.get('ARVADOS_API_HOST')
+ token = apiconfig.get('ARVADOS_API_TOKEN')
+ insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
+
+ return api(version=version, host=host, token=token, insecure=insecure, **kwargs)
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index dd48b99..b5eb6e6 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -2,7 +2,7 @@ import functools
import os
import zlib
import bz2
-from .ranges import *
+from ._ranges import locators_and_ranges, replace_range
from arvados.retry import retry_method
import config
import hashlib
@@ -13,6 +13,7 @@ import copy
import errno
from .errors import KeepWriteError, AssertionError
from .keep import KeepLocator
+from _normalize_stream import normalize_stream
def split(path):
"""Separate the stream name and file name in a /-separated stream path and
@@ -27,7 +28,7 @@ def split(path):
stream_name, file_name = '.', path
return stream_name, file_name
-class FileLikeObjectBase(object):
+class _FileLikeObjectBase(object):
def __init__(self, name, mode):
self.name = name
self.mode = mode
@@ -56,7 +57,7 @@ class FileLikeObjectBase(object):
self.closed = True
-class ArvadosFileReaderBase(FileLikeObjectBase):
+class ArvadosFileReaderBase(_FileLikeObjectBase):
class _NameAttribute(str):
# The Python file API provides a plain .name attribute.
# Older SDK provided a name() method.
@@ -214,7 +215,6 @@ class StreamFileReader(ArvadosFileReaderBase):
return ''.join(data)
def as_manifest(self):
- from stream import normalize_stream
segs = []
for r in self.segments:
segs.extend(self._stream.locators_and_ranges(r.locator, r.range_size))
@@ -228,7 +228,7 @@ def synchronized(orig_func):
return orig_func(self, *args, **kwargs)
return synchronized_wrapper
-class BufferBlock(object):
+class _BufferBlock(object):
"""A BufferBlock is a stand-in for a Keep block that is in the process of being
written.
@@ -269,7 +269,7 @@ class BufferBlock(object):
self.buffer_block = bytearray(starting_capacity)
self.buffer_view = memoryview(self.buffer_block)
self.write_pointer = 0
- self._state = BufferBlock.WRITABLE
+ self._state = _BufferBlock.WRITABLE
self._locator = None
self.owner = owner
self.lock = threading.Lock()
@@ -282,7 +282,7 @@ class BufferBlock(object):
buffer, doubling capacity as needed to accomdate all the data.
"""
- if self._state == BufferBlock.WRITABLE:
+ if self._state == _BufferBlock.WRITABLE:
while (self.write_pointer+len(data)) > len(self.buffer_block):
new_buffer_block = bytearray(len(self.buffer_block) * 2)
new_buffer_block[0:self.write_pointer] = self.buffer_block[0:self.write_pointer]
@@ -294,10 +294,15 @@ class BufferBlock(object):
else:
raise AssertionError("Buffer block is not writable")
- def set_state(self, nextstate):
- if ((self._state == BufferBlock.WRITABLE and nextstate == BufferBlock.PENDING) or
- (self._state == BufferBlock.PENDING and nextstate == BufferBlock.COMMITTED)):
+ @synchronized
+ def set_state(self, nextstate, loc=None):
+ if ((self._state == _BufferBlock.WRITABLE and nextstate == _BufferBlock.PENDING) or
+ (self._state == _BufferBlock.PENDING and nextstate == _BufferBlock.COMMITTED)):
self._state = nextstate
+ if self._state == _BufferBlock.COMMITTED:
+ bufferblock._locator = loc
+ bufferblock.buffer_view = None
+ bufferblock.buffer_block = None
else:
raise AssertionError("Invalid state change from %s to %s" % (self.state, state))
@@ -343,7 +348,7 @@ def must_be_writable(orig_func):
return must_be_writable_wrapper
-class BlockManager(object):
+class _BlockManager(object):
"""BlockManager handles buffer blocks, background block uploads, and background
block prefetch for a Collection of ArvadosFiles.
@@ -378,7 +383,7 @@ class BlockManager(object):
"""
if blockid is None:
blockid = "bufferblock%i" % len(self._bufferblocks)
- bufferblock = BufferBlock(blockid, starting_capacity=starting_capacity, owner=owner)
+ bufferblock = _BufferBlock(blockid, starting_capacity=starting_capacity, owner=owner)
self._bufferblocks[bufferblock.blockid] = bufferblock
return bufferblock
@@ -396,10 +401,10 @@ class BlockManager(object):
"""
new_blockid = "bufferblock%i" % len(self._bufferblocks)
with block.lock:
- if block._state == BufferBlock.COMMITTED:
+ if block._state == _BufferBlock.COMMITTED:
raise AssertionError("Can only duplicate a writable or pending buffer block")
- bufferblock = BufferBlock(new_blockid, block.size(), owner)
+ bufferblock = _BufferBlock(new_blockid, block.size(), owner)
bufferblock.append(block.buffer_view[0:block.size()])
self._bufferblocks[bufferblock.blockid] = bufferblock
return bufferblock
@@ -446,11 +451,7 @@ class BlockManager(object):
if bufferblock is None:
return
loc = self._keep.put(bufferblock.buffer_view[0:bufferblock.write_pointer].tobytes())
- with bufferblock.lock:
- bufferblock._locator = loc
- bufferblock.buffer_view = None
- bufferblock.buffer_block = None
- bufferblock.set_state(BufferBlock.COMMITTED)
+ bufferblock.set_state(_BufferBlock.COMMITTED, loc)
except Exception as e:
print e
@@ -483,8 +484,7 @@ class BlockManager(object):
thread.start()
# Mark the block as PENDING so to disallow any more appends.
- with block.lock:
- block.set_state(BufferBlock.PENDING)
+ block.set_state(_BufferBlock.PENDING)
self._put_queue.put(block)
@synchronized
@@ -501,7 +501,7 @@ class BlockManager(object):
with self.lock:
if locator in self._bufferblocks:
bufferblock = self._bufferblocks[locator]
- if bufferblock.state() != BufferBlock.COMMITTED:
+ if bufferblock.state() != _BufferBlock.COMMITTED:
return bufferblock.buffer_view[0:bufferblock.write_pointer].tobytes()
else:
locator = bufferblock._locator
@@ -522,7 +522,7 @@ class BlockManager(object):
items = self._bufferblocks.items()
for k,v in items:
- if v.state() == BufferBlock.WRITABLE:
+ if v.state() == _BufferBlock.WRITABLE:
self.commit_bufferblock(v)
with self.lock:
@@ -624,18 +624,18 @@ class ArvadosFile(object):
map_loc = {}
self._segments = []
- for r in other.segments():
- new_loc = r.locator
- if other.parent._my_block_manager().is_bufferblock(r.locator):
- if r.locator not in map_loc:
- bufferblock = other.parent._my_block_manager().get_bufferblock(r.locator)
- if bufferblock.state() != BufferBlock.WRITABLE:
- map_loc[r.locator] = bufferblock.locator()
+ for other_segment in other.segments():
+ new_loc = other_segment.locator
+ if other.parent._my_block_manager().is_bufferblock(other_segment.locator):
+ if other_segment.locator not in map_loc:
+ bufferblock = other.parent._my_block_manager().get_bufferblock(other_segment.locator)
+ if bufferblock.state() != _BufferBlock.WRITABLE:
+ map_loc[other_segment.locator] = bufferblock.locator()
else:
- map_loc[r.locator] = self.parent._my_block_manager().dup_block(bufferblock, self).blockid
- new_loc = map_loc[r.locator]
+ map_loc[other_segment.locator] = self.parent._my_block_manager().dup_block(bufferblock, self).blockid
+ new_loc = map_loc[other_segment.locator]
- self._segments.append(Range(new_loc, r.range_start, r.range_size, r.segment_offset))
+ self._segments.append(Range(new_loc, other_segment.range_start, other_segment.range_size, other_segment.segment_offset))
self._modified = True
@@ -779,7 +779,7 @@ class ArvadosFile(object):
self._modified = True
- if self._current_bblock is None or self._current_bblock.state() != BufferBlock.WRITABLE:
+ if self._current_bblock is None or self._current_bblock.state() != _BufferBlock.WRITABLE:
self._current_bblock = self.parent._my_block_manager().alloc_bufferblock(owner=self)
if (self._current_bblock.size() + len(data)) > config.KEEP_BLOCK_SIZE:
@@ -818,6 +818,25 @@ class ArvadosFile(object):
else:
return 0
+
+ @synchronized
+ def manifest_text(self, stream_name=".", portable_locators=False, normalize=False):
+ item = self
+ filestream = []
+ for segment in item.segments:
+ loc = segment.locator
+ if loc.startswith("bufferblock"):
+ loc = item._bufferblocks[loc].calculate_locator()
+ if portable_locators:
+ loc = KeepLocator(loc).stripped()
+ filestream.append(LocatorAndRange(loc, locator_block_size(loc),
+ segment.segment_offset, segment.range_size))
+ stream[stream_name] = filestream
+ buf += ' '.join(normalize_stream(stream_name, stream))
+ buf += "\n"
+ return buf
+
+
class ArvadosFileReader(ArvadosFileReaderBase):
"""Wraps ArvadosFile in a file-like object supporting reading only.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 41df555..82b9745 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -8,7 +8,7 @@ import time
from collections import deque
from stat import *
-from .arvfile import split, FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, BlockManager, synchronized, must_be_writable, SYNC_READONLY, SYNC_EXPLICIT, SYNC_LIVE, NoopLock
+from .arvfile import split, FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, _BlockManager, synchronized, must_be_writable, SYNC_READONLY, SYNC_EXPLICIT, NoopLock
from keep import *
from .stream import StreamReader, normalize_stream, locator_block_size
from .ranges import Range, LocatorAndRange
@@ -54,196 +54,6 @@ class CollectionBase(object):
return ''.join(clean)
-class CollectionReader(CollectionBase):
- def __init__(self, manifest_locator_or_text, api_client=None,
- keep_client=None, num_retries=0):
- """Instantiate a CollectionReader.
-
- This class parses Collection manifests to provide a simple interface
- to read its underlying files.
-
- Arguments:
- * manifest_locator_or_text: One of a Collection UUID, portable data
- hash, or full manifest text.
- * api_client: The API client to use to look up Collections. If not
- provided, CollectionReader will build one from available Arvados
- configuration.
- * keep_client: The KeepClient to use to download Collection data.
- If not provided, CollectionReader will build one from available
- Arvados configuration.
- * num_retries: The default number of times to retry failed
- service requests. Default 0. You may change this value
- after instantiation, but note those changes may not
- propagate to related objects like the Keep client.
- """
- self._api_client = api_client
- self._keep_client = keep_client
- self.num_retries = num_retries
- if re.match(util.keep_locator_pattern, manifest_locator_or_text):
- self._manifest_locator = manifest_locator_or_text
- self._manifest_text = None
- elif re.match(util.collection_uuid_pattern, manifest_locator_or_text):
- self._manifest_locator = manifest_locator_or_text
- self._manifest_text = None
- elif re.match(util.manifest_pattern, manifest_locator_or_text):
- self._manifest_text = manifest_locator_or_text
- self._manifest_locator = None
- else:
- raise errors.ArgumentError(
- "Argument to CollectionReader must be a manifest or a collection UUID")
- self._api_response = None
- self._streams = None
-
- def _populate_from_api_server(self):
- # As in KeepClient itself, we must wait until the last
- # possible moment to instantiate an API client, in order to
- # avoid tripping up clients that don't have access to an API
- # server. If we do build one, make sure our Keep client uses
- # it. If instantiation fails, we'll fall back to the except
- # clause, just like any other Collection lookup
- # failure. Return an exception, or None if successful.
- try:
- if self._api_client is None:
- self._api_client = arvados.api('v1')
- self._keep_client = None # Make a new one with the new api.
- self._api_response = self._api_client.collections().get(
- uuid=self._manifest_locator).execute(
- num_retries=self.num_retries)
- self._manifest_text = self._api_response['manifest_text']
- return None
- except Exception as e:
- return e
-
- def _populate_from_keep(self):
- # Retrieve a manifest directly from Keep. This has a chance of
- # working if [a] the locator includes a permission signature
- # or [b] the Keep services are operating in world-readable
- # mode. Return an exception, or None if successful.
- try:
- self._manifest_text = self._my_keep().get(
- self._manifest_locator, num_retries=self.num_retries)
- except Exception as e:
- return e
-
- def _populate(self):
- error_via_api = None
- error_via_keep = None
- should_try_keep = ((self._manifest_text is None) and
- util.keep_locator_pattern.match(
- self._manifest_locator))
- if ((self._manifest_text is None) and
- util.signed_locator_pattern.match(self._manifest_locator)):
- error_via_keep = self._populate_from_keep()
- if self._manifest_text is None:
- error_via_api = self._populate_from_api_server()
- if error_via_api is not None and not should_try_keep:
- raise error_via_api
- if ((self._manifest_text is None) and
- not error_via_keep and
- should_try_keep):
- # Looks like a keep locator, and we didn't already try keep above
- error_via_keep = self._populate_from_keep()
- if self._manifest_text is None:
- # Nothing worked!
- raise arvados.errors.NotFoundError(
- ("Failed to retrieve collection '{}' " +
- "from either API server ({}) or Keep ({})."
- ).format(
- self._manifest_locator,
- error_via_api,
- error_via_keep))
- self._streams = [sline.split()
- for sline in self._manifest_text.split("\n")
- if sline]
-
- def _populate_first(orig_func):
- # Decorator for methods that read actual Collection data.
- @functools.wraps(orig_func)
- def populate_first_wrapper(self, *args, **kwargs):
- if self._streams is None:
- self._populate()
- return orig_func(self, *args, **kwargs)
- return populate_first_wrapper
-
- @_populate_first
- def api_response(self):
- """api_response() -> dict or None
-
- Returns information about this Collection fetched from the API server.
- If the Collection exists in Keep but not the API server, currently
- returns None. Future versions may provide a synthetic response.
- """
- return self._api_response
-
- @_populate_first
- def normalize(self):
- # Rearrange streams
- streams = {}
- for s in self.all_streams():
- for f in s.all_files():
- streamname, filename = split(s.name() + "/" + f.name())
- if streamname not in streams:
- streams[streamname] = {}
- if filename not in streams[streamname]:
- streams[streamname][filename] = []
- for r in f.segments:
- streams[streamname][filename].extend(s.locators_and_ranges(r.locator, r.range_size))
-
- self._streams = [normalize_stream(s, streams[s])
- for s in sorted(streams)]
-
- # Regenerate the manifest text based on the normalized streams
- self._manifest_text = ''.join(
- [StreamReader(stream, keep=self._my_keep()).manifest_text()
- for stream in self._streams])
-
- @_populate_first
- def open(self, streampath, filename=None):
- """open(streampath[, filename]) -> file-like object
-
- Pass in the path of a file to read from the Collection, either as a
- single string or as two separate stream name and file name arguments.
- This method returns a file-like object to read that file.
- """
- if filename is None:
- streampath, filename = split(streampath)
- keep_client = self._my_keep()
- for stream_s in self._streams:
- stream = StreamReader(stream_s, keep_client,
- num_retries=self.num_retries)
- if stream.name() == streampath:
- break
- else:
- raise ValueError("stream '{}' not found in Collection".
- format(streampath))
- try:
- return stream.files()[filename]
- except KeyError:
- raise ValueError("file '{}' not found in Collection stream '{}'".
- format(filename, streampath))
-
- @_populate_first
- def all_streams(self):
- return [StreamReader(s, self._my_keep(), num_retries=self.num_retries)
- for s in self._streams]
-
- def all_files(self):
- for s in self.all_streams():
- for f in s.all_files():
- yield f
-
- @_populate_first
- def manifest_text(self, strip=False, normalize=False):
- if normalize:
- cr = CollectionReader(self.manifest_text())
- cr.normalize()
- return cr.manifest_text(strip=strip, normalize=False)
- elif strip:
- return self.stripped_manifest()
- else:
- return self._manifest_text
-
-
class _WriterFile(FileLikeObjectBase):
def __init__(self, coll_writer, name):
super(_WriterFile, self).__init__(name, 'wb')
@@ -715,9 +525,6 @@ class SynchronizedCollectionBase(CollectionBase):
"""
- if self.sync_mode() == SYNC_READONLY:
- raise IOError((errno.EROFS, "Collection is read only"))
-
pathcomponents = path.split("/")
if pathcomponents[0] == '.':
del pathcomponents[0]
@@ -1007,24 +814,50 @@ class SynchronizedCollectionBase(CollectionBase):
self.notify(ADD, target_dir, target_name, dup)
@synchronized
- def manifest_text(self, strip=False, normalize=False):
+ def manifest_text(self, stream_name=".", strip=False, normalize=False):
"""Get the manifest text for this collection, sub collections and files.
+ :stream_name:
+ Name of the stream (directory)
+
:strip:
If True, remove signing tokens from block locators if present.
- If False, block locators are left unchanged.
+ If False (default), block locators are left unchanged.
:normalize:
If True, always export the manifest text in normalized form
- even if the Collection is not modified. If False and the collection
+ even if the Collection is not modified. If False (default) and the collection
is not modified, return the original manifest text even if it is not
in normalized form.
"""
+
+ portable_locators = strip
if self.modified() or self._manifest_text is None or normalize:
- return export_manifest(self, stream_name=".", portable_locators=strip)
+ item = self
+ stream = {}
+ sorted_keys = sorted(item.keys())
+ for filename in [s for s in sorted_keys if isinstance(item[s], ArvadosFile)]:
+ # Create a stream per file `k`
+ arvfile = item[filename]
+ filestream = []
+ for segment in arvfile.segments():
+ loc = segment.locator
+ if arvfile.parent._my_block_manager().is_bufferblock(loc):
+ loc = arvfile.parent._my_block_manager().get_bufferblock(loc).locator()
+ if portable_locators:
+ loc = KeepLocator(loc).stripped()
+ filestream.append(LocatorAndRange(loc, locator_block_size(loc),
+ segment.segment_offset, segment.range_size))
+ stream[filename] = filestream
+ if stream:
+ buf += ' '.join(normalize_stream(stream_name, stream))
+ buf += "\n"
+ for dirname in [s for s in sorted_keys if isinstance(item[s], SynchronizedCollectionBase)]:
+ buf += item[dirname].manifest_text(stream_name=os.path.join(stream_name, dirname), portable_locators=portable_locators)
+ return buf
else:
- if strip:
+ if portable_locators:
return self.stripped_manifest()
else:
return self._manifest_text
@@ -1121,6 +954,7 @@ class SynchronizedCollectionBase(CollectionBase):
def __ne__(self, other):
return not self.__eq__(other)
+
class Collection(SynchronizedCollectionBase):
"""Represents the root of an Arvados Collection, which may be associated with
an API server Collection record.
@@ -1163,8 +997,7 @@ class Collection(SynchronizedCollectionBase):
api_client=None,
keep_client=None,
num_retries=None,
- block_manager=None,
- sync=None):
+ block_manager=None):
"""Collection constructor.
:manifest_locator_or_text:
@@ -1184,16 +1017,6 @@ class Collection(SynchronizedCollectionBase):
the number of retries for API and Keep requests.
:block_manager:
the block manager to use. If not specified, create one.
- :sync:
- Set synchronization policy with API server collection record.
- :SYNC_READONLY:
- Collection is read only. No synchronization. This mode will
- also forego locking, which gives better performance.
- :SYNC_EXPLICIT:
- Collection is writable. Synchronize on explicit request via `update()` or `save()`
- :SYNC_LIVE:
- Collection is writable. Synchronize with server in response to
- background websocket events, on block write, or on file close.
"""
super(Collection, self).__init__(parent)
@@ -1211,11 +1034,9 @@ class Collection(SynchronizedCollectionBase):
self._manifest_text = None
self._api_response = None
- if sync is None:
- raise errors.ArgumentError("Must specify sync mode")
-
- self._sync = sync
- self.lock = threading.RLock()
+ self._sync = SYNC_EXPLICIT
+ if not self.lock:
+ self.lock = threading.RLock()
self.callbacks = []
self.events = None
@@ -1231,7 +1052,6 @@ class Collection(SynchronizedCollectionBase):
"Argument to CollectionReader must be a manifest or a collection UUID")
self._populate()
- self._subscribe_events()
def root_collection(self):
@@ -1240,18 +1060,6 @@ class Collection(SynchronizedCollectionBase):
def sync_mode(self):
return self._sync
- def _subscribe_events(self):
- if self._sync == SYNC_LIVE and self.events is None:
- if not self._has_collection_uuid():
- raise errors.ArgumentError("Cannot SYNC_LIVE associated with a collection uuid")
- self.events = events.subscribe(arvados.api(apiconfig=self._config),
- [["object_uuid", "=", self._manifest_locator]],
- self.on_message)
-
- def on_message(self, event):
- if event.get("object_uuid") == self._manifest_locator:
- self.update()
-
@synchronized
@retry_method
def update(self, other=None, num_retries=None):
@@ -1286,7 +1094,7 @@ class Collection(SynchronizedCollectionBase):
@synchronized
def _my_block_manager(self):
if self._block_manager is None:
- self._block_manager = BlockManager(self._my_keep())
+ self._block_manager = _BlockManager(self._my_keep())
return self._block_manager
def _populate_from_api_server(self):
@@ -1350,10 +1158,6 @@ class Collection(SynchronizedCollectionBase):
self._baseline_manifest = self._manifest_text
import_manifest(self._manifest_text, self)
- if self._sync == SYNC_READONLY:
- # Now that we're populated, knowing that this will be readonly,
- # forego any further locking.
- self.lock = NoopLock()
def _has_collection_uuid(self):
return self._manifest_locator is not None and re.match(util.collection_uuid_pattern, self._manifest_locator)
@@ -1423,6 +1227,7 @@ class Collection(SynchronizedCollectionBase):
self._manifest_text = self._api_response["manifest_text"]
self.set_unmodified()
+
@must_be_writable
@synchronized
@retry_method
@@ -1431,7 +1236,7 @@ class Collection(SynchronizedCollectionBase):
a new collection record (if create_collection_record True).
After creating a new collection record, this Collection object will be
- associated with the new record for `save()` and SYNC_LIVE updates.
+ associated with the new record used by `save()`.
:name:
The collection name.
@@ -1465,14 +1270,8 @@ class Collection(SynchronizedCollectionBase):
self._api_response = self._my_api().collections().create(ensure_unique_name=ensure_unique_name, body=body).execute(num_retries=num_retries)
text = self._api_response["manifest_text"]
- if self.events:
- self.events.unsubscribe(filters=[["object_uuid", "=", self._manifest_locator]])
-
self._manifest_locator = self._api_response["uuid"]
- if self.events:
- self.events.subscribe(filters=[["object_uuid", "=", self._manifest_locator]])
-
self._manifest_text = text
self.set_unmodified()
@@ -1489,76 +1288,69 @@ class Collection(SynchronizedCollectionBase):
for c in self.callbacks:
c(event, collection, name, item)
-def ReadOnlyCollection(*args, **kwargs):
- """Create a read-only collection object from an api collection record locator,
- a portable data hash of a manifest, or raw manifest text.
-
- See `Collection` constructor for detailed options.
-
- """
- kwargs["sync"] = SYNC_READONLY
- return Collection(*args, **kwargs)
-
-def WritableCollection(*args, **kwargs):
- """Create a writable collection object from an api collection record locator,
- a portable data hash of a manifest, or raw manifest text.
-
- See `Collection` constructor for detailed options.
-
- """
-
- kwargs["sync"] = SYNC_EXPLICIT
- return Collection(*args, **kwargs)
-
-def LiveCollection(*args, **kwargs):
- """Create a writable, live updating collection object representing an existing
- collection record on the API server.
-
- See `Collection` constructor for detailed options.
-
- """
- kwargs["sync"] = SYNC_LIVE
- return Collection(*args, **kwargs)
-
-def createWritableCollection(name, owner_uuid=None, apiconfig=None):
- """Create an empty, writable collection object and create an associated api
- collection record.
-
- :name:
- The collection name
-
- :owner_uuid:
- The parent project.
+ @synchronized
+ def _import_manifest(self, manifest_text):
+ """Import a manifest into a `Collection`.
- :apiconfig:
- Optional alternate api configuration to use (to specify alternate API
- host or token than the default.)
+ :manifest_text:
+ The manifest text to import from.
- """
- newcollection = Collection(sync=SYNC_EXPLICIT, apiconfig=apiconfig)
- newcollection.save_new(name, owner_uuid=owner_uuid, ensure_unique_name=True)
- return newcollection
+ """
+ if len(self) > 0:
+ raise ArgumentError("Can only import manifest into an empty collection")
-def createLiveCollection(name, owner_uuid=None, apiconfig=None):
- """Create an empty, writable, live updating Collection object and create an
- associated collection record on the API server.
+ into_collection = self
+ save_sync = into_collection.sync_mode()
+ into_collection._sync = None
+
+ STREAM_NAME = 0
+ BLOCKS = 1
+ SEGMENTS = 2
+
+ stream_name = None
+ state = STREAM_NAME
+
+ for n in re.finditer(r'(\S+)(\s+|$)', manifest_text):
+ tok = n.group(1)
+ sep = n.group(2)
+
+ if state == STREAM_NAME:
+ # starting a new stream
+ stream_name = tok.replace('\\040', ' ')
+ blocks = []
+ segments = []
+ streamoffset = 0L
+ state = BLOCKS
+ continue
+
+ if state == BLOCKS:
+ s = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok)
+ if s:
+ blocksize = long(s.group(1))
+ blocks.append(Range(tok, streamoffset, blocksize))
+ streamoffset += blocksize
+ else:
+ state = SEGMENTS
+
+ if state == SEGMENTS:
+ s = re.search(r'^(\d+):(\d+):(\S+)', tok)
+ if s:
+ pos = long(s.group(1))
+ size = long(s.group(2))
+ name = s.group(3).replace('\\040', ' ')
+ f = into_collection.find_or_create("%s/%s" % (stream_name, name), FILE)
+ f.add_segment(blocks, pos, size)
+ else:
+ # error!
+ raise errors.SyntaxError("Invalid manifest format")
- :name:
- The collection name
+ if sep == "\n":
+ stream_name = None
+ state = STREAM_NAME
- :owner_uuid:
- The parent project.
+ into_collection.set_unmodified()
+ into_collection._sync = save_sync
- :apiconfig:
- Optional alternate api configuration to use (to specify alternate API
- host or token than the default.)
-
- """
- newcollection = Collection(sync=SYNC_EXPLICIT, apiconfig=apiconfig)
- newcollection.save_new(name, owner_uuid=owner_uuid, ensure_unique_name=True)
- newcollection._sync = SYNC_LIVE
- newcollection._subscribe_events()
- return newcollection
class Subcollection(SynchronizedCollectionBase):
"""This is a subdirectory within a collection that doesn't have its own API
@@ -1599,139 +1391,50 @@ class Subcollection(SynchronizedCollectionBase):
self._cloneinto(c)
return c
-def import_manifest(manifest_text,
- into_collection=None,
- api_client=None,
- keep=None,
- num_retries=None,
- sync=SYNC_READONLY):
- """Import a manifest into a `Collection`.
-
- :manifest_text:
- The manifest text to import from.
-
- :into_collection:
- The `Collection` that will be initialized (must be empty).
- If None, create a new `Collection` object.
- :api_client:
- The API client object that will be used when creating a new `Collection` object.
-
- :keep:
- The keep client object that will be used when creating a new `Collection` object.
+class CollectionReader(Collection):
+ """A read-only collection object from an api collection record locator,
+ a portable data hash of a manifest, or raw manifest text.
- :num_retries:
- the default number of api client and keep retries on error.
+ See `Collection` constructor for detailed options.
- :sync:
- Collection sync mode (only if into_collection is None)
"""
- if into_collection is not None:
- if len(into_collection) > 0:
- raise ArgumentError("Can only import manifest into an empty collection")
- else:
- into_collection = Collection(api_client=api_client, keep_client=keep, num_retries=num_retries, sync=sync)
-
- save_sync = into_collection.sync_mode()
- into_collection._sync = None
-
- STREAM_NAME = 0
- BLOCKS = 1
- SEGMENTS = 2
-
- stream_name = None
- state = STREAM_NAME
-
- for n in re.finditer(r'(\S+)(\s+|$)', manifest_text):
- tok = n.group(1)
- sep = n.group(2)
-
- if state == STREAM_NAME:
- # starting a new stream
- stream_name = tok.replace('\\040', ' ')
- blocks = []
- segments = []
- streamoffset = 0L
- state = BLOCKS
- continue
-
- if state == BLOCKS:
- s = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', tok)
- if s:
- blocksize = long(s.group(1))
- blocks.append(Range(tok, streamoffset, blocksize))
- streamoffset += blocksize
- else:
- state = SEGMENTS
-
- if state == SEGMENTS:
- s = re.search(r'^(\d+):(\d+):(\S+)', tok)
- if s:
- pos = long(s.group(1))
- size = long(s.group(2))
- name = s.group(3).replace('\\040', ' ')
- f = into_collection.find_or_create("%s/%s" % (stream_name, name), FILE)
- f.add_segment(blocks, pos, size)
- else:
- # error!
- raise errors.SyntaxError("Invalid manifest format")
+ def __init__(self, *args, **kwargs):
+ if not args and not kwargs.get("manifest_locator_or_text"):
+ raise errors.ArgumentError("Must provide manifest locator or text to initialize ReadOnlyCollection")
- if sep == "\n":
- stream_name = None
- state = STREAM_NAME
+ # Forego any locking since it should never change once initialized.
+ self.lock = NoopLock()
- into_collection.set_unmodified()
- into_collection._sync = save_sync
- return into_collection
+ super(ReadOnlyCollection, self).__init__(*args, **kwargs)
-def export_manifest(item, stream_name=".", portable_locators=False):
- """Export a manifest from the contents of a SynchronizedCollectionBase.
+ self._sync = SYNC_READONLY
- :item:
- Create a manifest for `item` (must be a `SynchronizedCollectionBase` or `ArvadosFile`). If
- `item` is a is a `Collection`, this will also export subcollections.
+ self._streams = [sline.split()
+ for sline in self._manifest_text.split("\n")
+ if sline]
- :stream_name:
- the name of the stream when exporting `item`.
+ def normalize(self):
+ # Rearrange streams
+ streams = {}
+ for s in self.all_streams():
+ for f in s.all_files():
+ streamname, filename = split(s.name() + "/" + f.name())
+ if streamname not in streams:
+ streams[streamname] = {}
+ if filename not in streams[streamname]:
+ streams[streamname][filename] = []
+ for r in f.segments:
+ streams[streamname][filename].extend(s.locators_and_ranges(r.locator, r.range_size))
- :portable_locators:
- If True, strip any permission hints on block locators.
- If False, use block locators as-is.
+ self._streams = [normalize_stream(s, streams[s])
+ for s in sorted(streams)]
- """
- buf = ""
- if isinstance(item, SynchronizedCollectionBase):
- stream = {}
- sorted_keys = sorted(item.keys())
- for filename in [s for s in sorted_keys if isinstance(item[s], ArvadosFile)]:
- # Create a stream per file `k`
- arvfile = item[filename]
- filestream = []
- for segment in arvfile.segments():
- loc = segment.locator
- if loc.startswith("bufferblock"):
- loc = arvfile.parent._my_block_manager()._bufferblocks[loc].locator()
- if portable_locators:
- loc = KeepLocator(loc).stripped()
- filestream.append(LocatorAndRange(loc, locator_block_size(loc),
- segment.segment_offset, segment.range_size))
- stream[filename] = filestream
- if stream:
- buf += ' '.join(normalize_stream(stream_name, stream))
- buf += "\n"
- for dirname in [s for s in sorted_keys if isinstance(item[s], SynchronizedCollectionBase)]:
- buf += export_manifest(item[dirname], stream_name=os.path.join(stream_name, dirname), portable_locators=portable_locators)
- elif isinstance(item, ArvadosFile):
- filestream = []
- for segment in item.segments:
- loc = segment.locator
- if loc.startswith("bufferblock"):
- loc = item._bufferblocks[loc].calculate_locator()
- if portable_locators:
- loc = KeepLocator(loc).stripped()
- filestream.append(LocatorAndRange(loc, locator_block_size(loc),
- segment.segment_offset, segment.range_size))
- stream[stream_name] = filestream
- buf += ' '.join(normalize_stream(stream_name, stream))
- buf += "\n"
- return buf
+ def all_streams(self):
+ return [StreamReader(s, self._my_keep(), num_retries=self.num_retries)
+ for s in self._streams]
+
+ def all_files(self):
+ for s in self.all_streams():
+ for f in s.all_files():
+ yield f
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index 32cb2a5..5c5c872 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -5,9 +5,11 @@ import config
import copy
class ThreadSafeApiCache(object):
- """Threadsafe wrapper for API objects. This stores and returns a different api
- object per thread, because httplib2 which underlies apiclient is not
- threadsafe.
+ """Threadsafe wrapper for API objects.
+
+ This stores and returns a different api object per thread, because httplib2
+ which underlies apiclient is not threadsafe.
+
"""
def __init__(self, apiconfig=None, keep_params={}):
@@ -19,7 +21,7 @@ class ThreadSafeApiCache(object):
def localapi(self):
if 'api' not in self.local.__dict__:
- self.local.api = api.api('v1', False, apiconfig=self.apiconfig)
+ self.local.api = api.api_from_config('v1', apiconfig=self.apiconfig)
return self.local.api
def __getattr__(self, name):
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index 9cfceb8..a6e0863 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -6,65 +6,13 @@ import threading
import functools
import copy
-from .ranges import *
+from ._ranges import locators_and_ranges
from .arvfile import StreamFileReader
from arvados.retry import retry_method
from keep import *
import config
import errors
-
-def locator_block_size(loc):
- s = re.match(r'[0-9a-f]{32}\+(\d+)(\+\S+)*', loc)
- return long(s.group(1))
-
-def normalize_stream(s, stream):
- '''
- s is the stream name
- stream is a dict mapping each filename to a list in the form [block locator, block size, segment offset (from beginning of block), segment size]
- returns the stream as a list of tokens
- '''
- stream_tokens = [s]
- sortedfiles = list(stream.keys())
- sortedfiles.sort()
-
- blocks = {}
- streamoffset = 0L
- # Go through each file and add each referenced block exactly once.
- for f in sortedfiles:
- for b in stream[f]:
- if b.locator not in blocks:
- stream_tokens.append(b.locator)
- blocks[b.locator] = streamoffset
- streamoffset += locator_block_size(b.locator)
-
- # Add the empty block if the stream is otherwise empty.
- if len(stream_tokens) == 1:
- stream_tokens.append(config.EMPTY_BLOCK_LOCATOR)
-
- for f in sortedfiles:
- # Add in file segments
- current_span = None
- fout = f.replace(' ', '\\040')
- for segment in stream[f]:
- # Collapse adjacent segments
- streamoffset = blocks[segment.locator] + segment.segment_offset
- if current_span is None:
- current_span = [streamoffset, streamoffset + segment.segment_size]
- else:
- if streamoffset == current_span[1]:
- current_span[1] += segment.segment_size
- else:
- stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
- current_span = [streamoffset, streamoffset + segment.segment_size]
-
- if current_span is not None:
- stream_tokens.append("{0}:{1}:{2}".format(current_span[0], current_span[1] - current_span[0], fout))
-
- if not stream[f]:
- stream_tokens.append("0:0:{0}".format(fout))
-
- return stream_tokens
-
+from _normalize_stream import normalize_stream
class StreamReader(object):
def __init__(self, tokens, keep=None, debug=False, _empty=False,
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 36f14c3..e5670e3 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -34,25 +34,25 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
class MockApi(object):
def __init__(self, b, r):
- self.b = b
- self.r = r
+ self.body = b
+ self.response = r
class MockCollections(object):
def __init__(self, b, r):
- self.b = b
- self.r = r
+ self.body = b
+ self.response = r
class Execute(object):
def __init__(self, r):
- self.r = r
+ self.response = r
def execute(self, num_retries=None):
- return self.r
+ return self.response
def create(self, ensure_unique_name=False, body=None):
- if body != self.b:
- raise Exception("Body %s does not match expectation %s" % (body, self.b))
- return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.r)
+ if body != self.body:
+ raise Exception("Body %s does not match expectation %s" % (body, self.body))
+ return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
def update(self, uuid=None, body=None):
- return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.r)
+ return ArvadosFileWriterTestCase.MockApi.MockCollections.Execute(self.response)
def collections(self):
- return ArvadosFileWriterTestCase.MockApi.MockCollections(self.b, self.r)
+ return ArvadosFileWriterTestCase.MockApi.MockCollections(self.body, self.response)
def test_truncate(self):
@@ -293,9 +293,11 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
{"uuid":"zzzzz-4zz18-mockcollection0",
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"})
with WritableCollection(api_client=api, keep_client=keep) as c:
+ self.assertIsNone(c.api_response())
writer = c.open("foo/bar/count.txt", "w+")
writer.write("01234567")
c.save_new("test_create")
+ self.assertEqual(c.api_response(), api.response)
def test_overwrite(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list