[ARVADOS] updated: 503c616f17ec4b77ce1778cc2033e6ed78927f0d
git at public.curoverse.com
git at public.curoverse.com
Tue Feb 17 12:13:22 EST 2015
Summary of changes:
sdk/python/arvados/__init__.py | 2 +-
sdk/python/arvados/arvfile.py | 125 ++++++++++++++++++++-----------
sdk/python/arvados/collection.py | 8 +-
sdk/python/tests/test_arvfile.py | 81 ++++++++++++--------
sdk/python/tests/test_collections.py | 141 ++++++++++++++++++++++++-----------
5 files changed, 234 insertions(+), 123 deletions(-)
via 503c616f17ec4b77ce1778cc2033e6ed78927f0d (commit)
from 5f905666581dd7ccd8f7e05d1c8c4a6eedff0da9 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 503c616f17ec4b77ce1778cc2033e6ed78927f0d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Feb 17 12:15:30 2015 -0500
4823: More tests and fixes for updating and merging from remote api record.
Tests pass.
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 59a1a44..e4d148e 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -19,7 +19,7 @@ import time
import threading
from api import *
-from collection import *
+from collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
from keep import *
from stream import *
from arvfile import StreamFileReader
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 0bc70a7..dd48b99 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -12,6 +12,7 @@ import Queue
import copy
import errno
from .errors import KeepWriteError, AssertionError
+from .keep import KeepLocator
def split(path):
"""Separate the stream name and file name in a /-separated stream path and
@@ -220,6 +221,13 @@ class StreamFileReader(ArvadosFileReaderBase):
return " ".join(normalize_stream(".", {self.name: segs})) + "\n"
+def synchronized(orig_func):
+ @functools.wraps(orig_func)
+ def synchronized_wrapper(self, *args, **kwargs):
+ with self.lock:
+ return orig_func(self, *args, **kwargs)
+ return synchronized_wrapper
+
class BufferBlock(object):
"""A BufferBlock is a stand-in for a Keep block that is in the process of being
written.
@@ -261,10 +269,12 @@ class BufferBlock(object):
self.buffer_block = bytearray(starting_capacity)
self.buffer_view = memoryview(self.buffer_block)
self.write_pointer = 0
- self.state = BufferBlock.WRITABLE
+ self._state = BufferBlock.WRITABLE
self._locator = None
self.owner = owner
+ self.lock = threading.Lock()
+ @synchronized
def append(self, data):
"""Append some data to the buffer.
@@ -272,7 +282,7 @@ class BufferBlock(object):
buffer, doubling capacity as needed to accomdate all the data.
"""
- if self.state == BufferBlock.WRITABLE:
+ if self._state == BufferBlock.WRITABLE:
while (self.write_pointer+len(data)) > len(self.buffer_block):
new_buffer_block = bytearray(len(self.buffer_block) * 2)
new_buffer_block[0:self.write_pointer] = self.buffer_block[0:self.write_pointer]
@@ -284,10 +294,22 @@ class BufferBlock(object):
else:
raise AssertionError("Buffer block is not writable")
+ def set_state(self, nextstate):
+ if ((self._state == BufferBlock.WRITABLE and nextstate == BufferBlock.PENDING) or
+ (self._state == BufferBlock.PENDING and nextstate == BufferBlock.COMMITTED)):
+ self._state = nextstate
+ else:
+ raise AssertionError("Invalid state change from %s to %s" % (self.state, state))
+
+ @synchronized
+ def state(self):
+ return self._state
+
def size(self):
"""The amount of data written to the buffer."""
return self.write_pointer
+ @synchronized
def locator(self):
"""The Keep locator for this buffer's contents."""
if self._locator is None:
@@ -295,13 +317,6 @@ class BufferBlock(object):
return self._locator
-def synchronized(orig_func):
- @functools.wraps(orig_func)
- def synchronized_wrapper(self, *args, **kwargs):
- with self.lock:
- return orig_func(self, *args, **kwargs)
- return synchronized_wrapper
-
class NoopLock(object):
def __enter__(self):
return self
@@ -368,24 +383,24 @@ class BlockManager(object):
return bufferblock
@synchronized
- def dup_block(self, blockid, owner):
+ def dup_block(self, block, owner):
"""Create a new bufferblock in WRITABLE state, initialized with the content of
an existing bufferblock.
- :blockid:
- the block to copy. May be an existing buffer block id.
+ :block:
+ the buffer block to copy.
:owner:
ArvadosFile that owns the new block
"""
new_blockid = "bufferblock%i" % len(self._bufferblocks)
- block = self._bufferblocks[blockid]
- if block.state != BufferBlock.WRITABLE:
- raise AssertionError("Can only duplicate a writable buffer block")
+ with block.lock:
+ if block._state == BufferBlock.COMMITTED:
+ raise AssertionError("Can only duplicate a writable or pending buffer block")
- bufferblock = BufferBlock(new_blockid, block.size(), owner)
- bufferblock.append(block.buffer_view[0:block.size()])
+ bufferblock = BufferBlock(new_blockid, block.size(), owner)
+ bufferblock.append(block.buffer_view[0:block.size()])
self._bufferblocks[bufferblock.blockid] = bufferblock
return bufferblock
@@ -430,10 +445,13 @@ class BlockManager(object):
bufferblock = self._put_queue.get()
if bufferblock is None:
return
- bufferblock._locator = self._keep.put(bufferblock.buffer_view[0:bufferblock.write_pointer].tobytes())
- bufferblock.state = BufferBlock.COMMITTED
- bufferblock.buffer_view = None
- bufferblock.buffer_block = None
+ loc = self._keep.put(bufferblock.buffer_view[0:bufferblock.write_pointer].tobytes())
+ with bufferblock.lock:
+ bufferblock._locator = loc
+ bufferblock.buffer_view = None
+ bufferblock.buffer_block = None
+ bufferblock.set_state(BufferBlock.COMMITTED)
+
except Exception as e:
print e
self._put_errors.put((bufferblock.locator(), e))
@@ -465,12 +483,13 @@ class BlockManager(object):
thread.start()
# Mark the block as PENDING so to disallow any more appends.
- block.state = BufferBlock.PENDING
+ with block.lock:
+ block.set_state(BufferBlock.PENDING)
self._put_queue.put(block)
+ @synchronized
def get_bufferblock(self, locator):
- with self.lock:
- return self._bufferblocks.get(locator)
+ return self._bufferblocks.get(locator)
def get_block_contents(self, locator, num_retries, cache_only=False):
"""Fetch a block.
@@ -482,7 +501,7 @@ class BlockManager(object):
with self.lock:
if locator in self._bufferblocks:
bufferblock = self._bufferblocks[locator]
- if bufferblock.state != BufferBlock.COMMITTED:
+ if bufferblock.state() != BufferBlock.COMMITTED:
return bufferblock.buffer_view[0:bufferblock.write_pointer].tobytes()
else:
locator = bufferblock._locator
@@ -503,12 +522,13 @@ class BlockManager(object):
items = self._bufferblocks.items()
for k,v in items:
- if v.state == BufferBlock.WRITABLE:
+ if v.state() == BufferBlock.WRITABLE:
self.commit_bufferblock(v)
with self.lock:
if self._put_queue is not None:
self._put_queue.join()
+
if not self._put_errors.empty():
err = []
try:
@@ -594,28 +614,29 @@ class ArvadosFile(object):
def clone(self, new_parent):
"""Make a copy of this file."""
cp = ArvadosFile(new_parent)
+ cp.replace_contents(self)
+ return cp
+
+ @must_be_writable
+ @synchronized
+ def replace_contents(self, other):
+ """Replace segments of this file with segments from another `ArvadosFile` object."""
map_loc = {}
- for r in self._segments:
+ self._segments = []
+ for r in other.segments():
new_loc = r.locator
- if self.parent._my_block_manager().is_bufferblock(r.locator):
+ if other.parent._my_block_manager().is_bufferblock(r.locator):
if r.locator not in map_loc:
- bufferblock = get_bufferblock(r.locator)
- if bufferblock.state == BufferBlock.COMITTED:
+ bufferblock = other.parent._my_block_manager().get_bufferblock(r.locator)
+ if bufferblock.state() != BufferBlock.WRITABLE:
map_loc[r.locator] = bufferblock.locator()
else:
- map_loc[r.locator] = self.parent._my_block_manager().dup_block(r.locator, cp)
+ map_loc[r.locator] = self.parent._my_block_manager().dup_block(bufferblock, self).blockid
new_loc = map_loc[r.locator]
- cp._segments.append(Range(new_loc, r.range_start, r.range_size, r.segment_offset))
+ self._segments.append(Range(new_loc, r.range_start, r.range_size, r.segment_offset))
- return cp
-
- @must_be_writable
- @synchronized
- def replace_contents(self, other):
- """Replace segments of this file with segments from another `ArvadosFile` object."""
- self._segments = other.segments()
self._modified = True
def __eq__(self, other):
@@ -624,9 +645,29 @@ class ArvadosFile(object):
if not isinstance(other, ArvadosFile):
return False
- s = other.segments()
+ othersegs = other.segments()
with self.lock:
- return self._segments == s
+ if len(self._segments) != len(othersegs):
+ return False
+ for i in xrange(0, len(othersegs)):
+ seg1 = self._segments[i]
+ seg2 = othersegs[i]
+ loc1 = seg1.locator
+ loc2 = seg2.locator
+
+ if self.parent._my_block_manager().is_bufferblock(loc1):
+ loc1 = self.parent._my_block_manager().get_bufferblock(loc1).locator()
+
+ if other.parent._my_block_manager().is_bufferblock(loc2):
+ loc2 = other.parent._my_block_manager().get_bufferblock(loc2).locator()
+
+ if (KeepLocator(loc1).stripped() != KeepLocator(loc2).stripped() or
+ seg1.range_start != seg2.range_start or
+ seg1.range_size != seg2.range_size or
+ seg1.segment_offset != seg2.segment_offset):
+ return False
+
+ return True
def __ne__(self, other):
return not self.__eq__(other)
@@ -738,7 +779,7 @@ class ArvadosFile(object):
self._modified = True
- if self._current_bblock is None or self._current_bblock.state != BufferBlock.WRITABLE:
+ if self._current_bblock is None or self._current_bblock.state() != BufferBlock.WRITABLE:
self._current_bblock = self.parent._my_block_manager().alloc_bufferblock(owner=self)
if (self._current_bblock.size() + len(data)) > config.KEEP_BLOCK_SIZE:
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 33af0c2..f6dc4d5 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1037,7 +1037,7 @@ class SynchronizedCollectionBase(CollectionBase):
"""
changes = []
if holding_collection is None:
- holding_collection = Collection(api_client=self._my_api(), keep_client=self._my_keep(), sync=SYNC_READONLY)
+ holding_collection = Collection(api_client=self._my_api(), keep_client=self._my_keep(), sync=SYNC_EXPLICIT)
for k in self:
if k not in end_collection:
changes.append((DEL, os.path.join(prefix, k), self[k].clone(holding_collection)))
@@ -1265,7 +1265,7 @@ class Collection(SynchronizedCollectionBase):
response = self._my_api().collections().get(uuid=self._manifest_locator).execute(num_retries=num_retries)
other = import_manifest(response["manifest_text"])
baseline = import_manifest(self._manifest_text)
- self.apply(other.diff(baseline))
+ self.apply(baseline.diff(other))
@synchronized
def _my_api(self):
@@ -1372,11 +1372,11 @@ class Collection(SynchronizedCollectionBase):
def clone(self, new_parent=None, new_sync=SYNC_READONLY, new_config=None):
if new_config is None:
new_config = self._config
- newcollection = Collection(parent=new_parent, apiconfig=new_config, sync=new_sync)
+ newcollection = Collection(parent=new_parent, apiconfig=new_config, sync=SYNC_EXPLICIT)
if new_sync == SYNC_READONLY:
newcollection.lock = NoopLock()
- newcollection._items = {}
self._cloneinto(newcollection)
+ newcollection._sync = new_sync
return newcollection
@synchronized
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 29775a6..aaf79a2 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -9,8 +9,9 @@ import unittest
import hashlib
import arvados
-from arvados import ArvadosFile, ArvadosFileReader, Range, import_manifest, export_manifest, KeepLocator
-from arvados.arvfile import SYNC_READONLY, SYNC_EXPLICIT
+from arvados import Range, KeepLocator
+from arvados.collection import import_manifest, export_manifest, ReadOnlyCollection, WritableCollection
+from arvados.arvfile import ArvadosFile, ArvadosFileReader, SYNC_READONLY, SYNC_EXPLICIT
import arvados_testutil as tutil
from test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
@@ -59,7 +60,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
"manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
@@ -80,7 +81,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
"manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual(writer.size(), 10)
@@ -107,7 +108,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_append1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
writer = c.open("count.txt", "a+")
self.assertEqual(writer.read(20), "0123456789")
writer.seek(0, os.SEEK_SET)
@@ -123,7 +124,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write0(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
@@ -135,7 +136,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
@@ -147,7 +148,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write2(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("0123456789", writer.readfrom(0, 13))
@@ -159,7 +160,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write3(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012345678901234", writer.readfrom(0, 15))
@@ -171,7 +172,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write4(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
self.assertEqual("012301230123", writer.readfrom(0, 15))
@@ -186,7 +187,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
"manifest_text": ". a5de24f4417cfba9d5825eadc2f4ca49+67108000 598cc1a4ccaef8ab6e4724d87e675d78+32892000 0:100000000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
@@ -202,7 +203,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_rewrite0(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
- with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
@@ -215,7 +216,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_rewrite1(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
@@ -228,7 +229,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_rewrite2(self):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
writer = c.open("count.txt", "r+")
for b in xrange(0, 10):
@@ -244,7 +245,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_write_large",
"manifest_text": ". 37400a68af9abdd76ca5bf13e819e42a+32892003 a5de24f4417cfba9d5825eadc2f4ca49+67108000 32892000:3:count.txt 32892006:67107997:count.txt 0:32892000:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
+ with WritableCollection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "r+")
text = ''.join(["0123456789" for a in xrange(0, 100)])
@@ -265,7 +266,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
+ with WritableCollection(api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
@@ -285,7 +286,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create",
"manifest_text":"./foo/bar 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
+ with WritableCollection(api_client=api, keep_client=keep) as c:
writer = c.open("foo/bar/count.txt", "w+")
writer.write("01234567")
c.save_new("test_create")
@@ -295,7 +296,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_overwrite",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 0:8:count.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "w+")
self.assertEqual(writer.size(), 0)
@@ -309,12 +310,12 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertEqual(False, c.modified())
def test_file_not_found(self):
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open("nocount.txt", "r")
def test_cannot_open_directory(self):
- with arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
writer = c.open(".", "r")
@@ -323,7 +324,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({"name":"test_create_multiple",
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n"},
{"uuid":"zzzzz-4zz18-mockcollection0"})
- with arvados.WritableCollection(api_client=api, keep_client=keep) as c:
+ with WritableCollection(api_client=api, keep_client=keep) as c:
w1 = c.open("count1.txt", "w")
w2 = c.open("count2.txt", "w")
w1.write("01234567")
@@ -399,18 +400,36 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
def test_prefetch(self):
keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})
- with arvados.WritableCollection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
+ with WritableCollection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
r = c.open("count.txt", "r")
self.assertEqual("0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
- def test__eq__(self):
- with arvados.arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
- with arvados.arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
+ def test__eq__1(self):
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
+ def test__eq__2(self):
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with WritableCollection() as c2:
+ with c2.open("count1.txt", "w") as f:
+ f.write("0123456789")
+
+ self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
+ self.assertFalse(c1["count1.txt"] != c2["count1.txt"])
+
+ def test__ne__(self):
+ with WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
+ with WritableCollection() as c2:
+ with c2.open("count1.txt", "w") as f:
+ f.write("1234567890")
+
+ self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
+ self.assertFalse(c1["count1.txt"] == c2["count1.txt"])
+
class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
def reader_for(self, coll_name, **kwargs):
@@ -425,7 +444,7 @@ class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
n += k.size
except ValueError:
pass
- col = arvados.ReadOnlyCollection(keep_client=self.keep_client())
+ col = ReadOnlyCollection(keep_client=self.keep_client())
col._my_block_manager().prefetch_enabled = False
af = ArvadosFile(col,
stream=stream,
@@ -472,7 +491,7 @@ class BlockManagerTest(unittest.TestCase):
self.assertEqual(bufferblock.buffer_view[0:6], "foobar")
self.assertEqual(bufferblock.locator(), "3858f62230ac3c915f300c664312c63f+6")
- bufferblock.state = arvados.arvfile.BufferBlock.PENDING
+ bufferblock.set_state(arvados.arvfile.BufferBlock.PENDING)
with self.assertRaises(arvados.errors.AssertionError):
bufferblock.append("bar")
@@ -485,9 +504,9 @@ class BlockManagerTest(unittest.TestCase):
self.assertEqual(bufferblock.size(), 3)
self.assertEqual(bufferblock.buffer_view[0:3], "foo")
self.assertEqual(bufferblock.locator(), "acbd18db4cc2f85cedef654fccc4a4d8+3")
- bufferblock.state = arvados.arvfile.BufferBlock.PENDING
+ bufferblock.set_state(arvados.arvfile.BufferBlock.PENDING)
- bufferblock2 = blockmanager.dup_block(bufferblock.blockid, None)
+ bufferblock2 = blockmanager.dup_block(bufferblock, None)
self.assertNotEqual(bufferblock.blockid, bufferblock2.blockid)
bufferblock2.append("bar")
@@ -516,7 +535,7 @@ class BlockManagerTest(unittest.TestCase):
bufferblock.append("foo")
blockmanager.commit_all()
self.assertTrue(mockkeep.put.called)
- self.assertEqual(bufferblock.state, arvados.arvfile.BufferBlock.COMMITTED)
+ self.assertEqual(bufferblock.state(), arvados.arvfile.BufferBlock.COMMITTED)
self.assertIsNone(bufferblock.buffer_view)
@@ -529,4 +548,4 @@ class BlockManagerTest(unittest.TestCase):
with self.assertRaises(arvados.errors.KeepWriteError) as err:
blockmanager.commit_all()
self.assertEquals(str(err.exception), "Error writing some blocks: acbd18db4cc2f85cedef654fccc4a4d8+3 raised KeepWriteError (fail)")
- self.assertEqual(bufferblock.state, arvados.arvfile.BufferBlock.PENDING)
+ self.assertEqual(bufferblock.state(), arvados.arvfile.BufferBlock.PENDING)
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 5e50357..a0cdb0d 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -15,7 +15,7 @@ import unittest
import run_test_server
import arvados_testutil as tutil
from arvados.ranges import Range, LocatorAndRange
-from arvados import import_manifest, export_manifest
+from arvados.collection import import_manifest, export_manifest, ReadOnlyCollection, WritableCollection
from arvados.arvfile import SYNC_EXPLICIT
class TestResumableWriter(arvados.ResumableCollectionWriter):
@@ -821,18 +821,18 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.import_manifest(m1)))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", export_manifest(import_manifest(m1)))
def test_init_manifest(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.ReadOnlyCollection(m1)))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", export_manifest(ReadOnlyCollection(m1)))
def test_remove(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n')
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
self.assertIn("count1.txt", c)
c.remove("count1.txt")
@@ -840,51 +840,51 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
def test_remove_in_subdir(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_remove_empty_subdir(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
c.remove("foo/count2.txt")
c.remove("foo")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_remove_nonempty_subdir(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
with self.assertRaises(IOError):
c.remove("foo")
c.remove("foo", recursive=True)
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_to_dir1(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
def test_copy_to_dir2(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_to_dir2(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "foo/")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_file(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
c.copy("count1.txt", "count2.txt")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_clone(self):
- c = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
cl = c.clone()
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
def test_diff1(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
('add', './count1.txt', c1["count1.txt"])])
@@ -896,8 +896,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff2(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [])
d = c1.diff(c2)
@@ -908,8 +908,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff3(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
d = c1.diff(c2)
@@ -920,8 +920,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff4(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
d = c1.diff(c2)
@@ -932,8 +932,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff5(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo', c2["foo"])])
d = c1.diff(c2)
@@ -944,8 +944,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff6(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
- c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
@@ -959,8 +959,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff7(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
- c2 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c2 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
d = c1.diff(c2)
@@ -971,8 +971,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_conflict1(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
- c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
+ c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
('add', './count2.txt', c2["count2.txt"])])
@@ -984,8 +984,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), ". 95ebc3c7b3b9f1d2c40fec14415d3cb8+5 5348b82a029fd9e971a811ce1f71360b+43 0:5:count1.txt 5:10:count2.txt\n")
def test_conflict2(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
with c1.open("count1.txt", "w") as f:
@@ -997,8 +997,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.manifest_text()))
def test_conflict3(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
- c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n')
+ c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c1.diff(c2)
self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]),
('add', './count1.txt', c2["count1.txt"])])
@@ -1011,8 +1011,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.manifest_text()))
def test_conflict4(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- c2 = arvados.WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c2 = WritableCollection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
c1.remove("count1.txt")
@@ -1023,17 +1023,17 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.manifest_text()))
def test_notify1(self):
- c1 = arvados.WritableCollection()
+ c1 = WritableCollection()
events = []
c1.subscribe(lambda event, collection, name, item: events.append((event, collection, name, item)))
f = c1.open("foo.txt", "w")
self.assertEqual(events[0], (arvados.collection.ADD, c1, "foo.txt", f.arvadosfile))
def test_open_w(self):
- c1 = arvados.WritableCollection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- self.assertEqual(c1["count.txt"].size(), 10)
- c1.open("count.txt", "w").close()
- self.assertEqual(c1["count.txt"].size(), 0)
+ c1 = WritableCollection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
+ self.assertEqual(c1["count1.txt"].size(), 10)
+ c1.open("count1.txt", "w").close()
+ self.assertEqual(c1["count1.txt"].size(), 0)
class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
@@ -1056,6 +1056,40 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
self.assertTrue(re.match(r"^\. 781e5e245d69b566979b86e28d23f2c7\+10\+A[a-f0-9]{40}@[a-f0-9]{8} 0:10:count.txt$",
c2["manifest_text"]))
+ def test_create_diff_apply(self):
+ c1 = arvados.collection.createWritableCollection("hello world")
+ self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
+ self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
+ with c1.open("count.txt", "w") as f:
+ f.write("0123456789")
+
+ self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
+
+ c1.save()
+
+ c2 = WritableCollection(c1._manifest_locator)
+ with c2.open("count.txt", "w") as f:
+ f.write("abcdefg")
+
+ diff = c1.diff(c2)
+
+ self.assertEqual(diff[0], (arvados.collection.MOD, u'./count.txt', c1["count.txt"], c2["count.txt"]))
+
+ c1.apply(diff)
+ self.assertEqual(c1.portable_data_hash(), c2.portable_data_hash())
+
+ def test_diff_apply_with_token(self):
+ baseline = ReadOnlyCollection(". 781e5e245d69b566979b86e28d23f2c7+10+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:10:count.txt\n")
+ c = WritableCollection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
+ other = ReadOnlyCollection(". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:7:count.txt\n")
+
+ diff = baseline.diff(other)
+ self.assertEqual(diff, [('mod', u'./count.txt', c["count.txt"], other["count.txt"])])
+
+ c.apply(diff)
+
+ self.assertEqual(c.manifest_text(), ". 7ac66c0f148de9519b8bd264312c4d64+7+A715fd31f8111894f717eb1003c1b0216799dd9ec at 54f5dd1a 0:7:count.txt\n")
+
def test_create_and_update(self):
c1 = arvados.collection.createWritableCollection("hello world")
@@ -1066,9 +1100,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
- print c1.manifest_text()
c1.save()
- print c1.manifest_text()
c2 = arvados.collection.WritableCollection(c1._manifest_locator)
with c2.open("count.txt", "w") as f:
@@ -1077,13 +1109,32 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c2.save()
self.assertNotEqual(c1.portable_data_hash(), c2.portable_data_hash())
-
- print c1.manifest_text()
c1.update()
- print c1.manifest_text()
-
self.assertEqual(c1.portable_data_hash(), c2.portable_data_hash())
+ def test_create_and_update_with_conflict(self):
+ c1 = arvados.collection.createWritableCollection("hello world")
+ self.assertEquals(c1.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
+ self.assertEquals(c1.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
+ with c1.open("count.txt", "w") as f:
+ f.write("0123456789")
+
+ self.assertEquals(c1.manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
+
+ c1.save()
+ with c1.open("count.txt", "w") as f:
+ f.write("XYZ")
+
+ c2 = arvados.collection.WritableCollection(c1._manifest_locator)
+ with c2.open("count.txt", "w") as f:
+ f.write("abcdefg")
+
+ c2.save()
+
+ c1.update()
+ self.assertTrue(re.match(r"\. e65075d550f9b5bf9992fa1d71a131be\+3 7ac66c0f148de9519b8bd264312c4d64\+7\+A[a-f0-9]{40}@[a-f0-9]{8} 0:3:count.txt 3:7:count.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~$", c1.manifest_text()))
+
+
if __name__ == '__main__':
unittest.main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list