[ARVADOS] updated: bac1bf1fa5154b0db653d5ba0353458e1fc24dd2
git at public.curoverse.com
git at public.curoverse.com
Thu Feb 5 11:20:13 EST 2015
Summary of changes:
sdk/python/arvados/arvfile.py | 28 ++-----
sdk/python/arvados/collection.py | 85 ++++++++++++++-------
sdk/python/tests/test_collections.py | 138 +++++++++++++++++++++++------------
3 files changed, 158 insertions(+), 93 deletions(-)
via bac1bf1fa5154b0db653d5ba0353458e1fc24dd2 (commit)
from 5c0430fd80db2fd3d57243d9187588c6854a738b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit bac1bf1fa5154b0db653d5ba0353458e1fc24dd2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Feb 5 11:22:23 2015 -0500
4823: Files now fall under collection's lock. Add tests for merge conflicts.
Add modified flag to collections to detect deletions.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 97d0605..2dc2b2c 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -566,10 +566,7 @@ class ArvadosFile(object):
for s in segments:
self._add_segment(stream, s.locator, s.range_size)
self._current_bblock = None
- if parent.sync_mode() == SYNC_READONLY:
- self.lock = NoopLock()
- else:
- self.lock = threading.Lock()
+ self.lock = parent._root_lock()
def sync_mode(self):
return self.parent.sync_mode()
@@ -582,7 +579,6 @@ class ArvadosFile(object):
def clone(self, new_parent):
"""Make a copy of this file."""
cp = ArvadosFile(new_parent)
- cp._modified = False
map_loc = {}
for r in self._segments:
@@ -606,7 +602,7 @@ class ArvadosFile(object):
def __eq__(self, other):
if other is self:
return True
- if type(other) != ArvadosFile:
+ if not isinstance(other, ArvadosFile):
return False
s = other.segments()
@@ -634,7 +630,7 @@ class ArvadosFile(object):
the file contents after `size` will be discarded. If `size` is greater
than the current size of the file, an IOError will be raised.
"""
- if size < self._size():
+ if size < self.size():
new_segs = []
for r in self._segments:
range_end = r.range_start+r.range_size
@@ -651,7 +647,7 @@ class ArvadosFile(object):
self._segments = new_segs
self._modified = True
- elif size > self._size():
+ elif size > self.size():
raise IOError("truncate() does not support extending the file size")
@_synchronized
@@ -659,7 +655,7 @@ class ArvadosFile(object):
"""
read upto `size` bytes from the file starting at `offset`.
"""
- if size == 0 or offset >= self._size():
+ if size == 0 or offset >= self.size():
return ''
data = []
@@ -708,7 +704,7 @@ class ArvadosFile(object):
if len(data) == 0:
return
- if offset > self._size():
+ if offset > self.size():
raise ArgumentError("Offset is past the end of the file")
if len(data) > config.KEEP_BLOCK_SIZE:
@@ -732,10 +728,6 @@ class ArvadosFile(object):
@_must_be_writable
@_synchronized
def add_segment(self, blocks, pos, size):
- # Synchronized public api, see _add_segment
- self._add_segment(blocks, pos, size)
-
- def _add_segment(self, blocks, pos, size):
"""
Add a segment to the end of the file, with `pos` and `offset` referencing a
section of the stream described by `blocks` (a list of Range objects)
@@ -746,7 +738,8 @@ class ArvadosFile(object):
r = Range(lr.locator, last.range_start+last.range_size, lr.segment_size, lr.segment_offset)
self._segments.append(r)
- def _size(self):
+ @_synchronized
+ def size(self):
"""Get the file size"""
if self._segments:
n = self._segments[-1]
@@ -754,11 +747,6 @@ class ArvadosFile(object):
else:
return 0
- @_synchronized
- def size(self):
- """Get the file size"""
- return self._size()
-
class ArvadosFileReader(ArvadosFileReaderBase):
def __init__(self, arvadosfile, name, mode="r", num_retries=None):
super(ArvadosFileReader, self).__init__(name, mode, num_retries=num_retries)
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 3b23143..f5ed2f3 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -649,6 +649,7 @@ MOD = "mod"
class SynchronizedCollectionBase(CollectionBase):
def __init__(self, parent=None):
self.parent = parent
+ self._modified = True
self._items = {}
def _my_api(self):
@@ -669,7 +670,7 @@ class SynchronizedCollectionBase(CollectionBase):
def sync_mode(self):
raise NotImplementedError()
- def notify(self, collection, event, name, item):
+ def notify(self, event, collection, name, item):
raise NotImplementedError()
@_synchronized
@@ -708,14 +709,16 @@ class SynchronizedCollectionBase(CollectionBase):
else:
item = ArvadosFile(self)
self._items[p[0]] = item
- self.notify(self, ADD, p[0], item)
+ self._modified = True
+ self.notify(ADD, self, p[0], item)
return item
else:
if item is None and create:
# create new collection
item = Subcollection(self)
self._items[p[0]] = item
- self.notify(self, ADD, p[0], item)
+ self._modified = True
+ self.notify(ADD, self, p[0], item)
del p[0]
if isinstance(item, SynchronizedCollectionBase):
return item.find("/".join(p), create=create)
@@ -769,6 +772,8 @@ class SynchronizedCollectionBase(CollectionBase):
def modified(self):
"""Test if the collection (or any subcollection or file) has been modified
since it was created."""
+ if self._modified:
+ return True
for k,v in self._items.items():
if v.modified():
return True
@@ -777,6 +782,7 @@ class SynchronizedCollectionBase(CollectionBase):
@_synchronized
def set_unmodified(self):
"""Recursively clear modified flag"""
+ self._modified = False
for k,v in self._items.items():
v.set_unmodified()
@@ -813,7 +819,8 @@ class SynchronizedCollectionBase(CollectionBase):
def __delitem__(self, p):
"""Delete an item by name which is directly contained by this collection."""
del self._items[p]
- self.notify(self, DEL, p, None)
+ self._modified = True
+ self.notify(DEL, self, p, None)
@_synchronized
def keys(self):
@@ -853,8 +860,10 @@ class SynchronizedCollectionBase(CollectionBase):
if len(p) == 1:
if isinstance(self._items[p[0]], SynchronizedCollectionBase) and len(self._items[p[0]]) > 0 and not rm_r:
raise IOError((errno.ENOTEMPTY, "Subcollection not empty"))
+ d = self._items[p[0]]
del self._items[p[0]]
- self.notify(self, DEL, p[0], None)
+ self._modified = True
+ self.notify(DEL, self, p[0], d)
else:
del p[0]
item.remove("/".join(p))
@@ -912,20 +921,27 @@ class SynchronizedCollectionBase(CollectionBase):
target_dir = self.find("/".join(tp[0:-1]), create=True, create_collection=True)
- if target_name in target_dir:
- if isinstance(target_dir[target_name], SynchronizedCollectionBase) and sp:
- target_dir = target_dir[target_name]
- target_name = sp[-1]
- elif not overwrite:
- raise IOError((errno.EEXIST, "File already exists"))
-
- # Actually make the copy.
- dup = source_obj.clone(target_dir)
with target_dir.lock:
+ if target_name in target_dir:
+ if isinstance(target_dir[target_name], SynchronizedCollectionBase) and sp:
+ target_dir = target_dir[target_name]
+ target_name = sp[-1]
+ elif not overwrite:
+ raise IOError((errno.EEXIST, "File already exists"))
+
+ mod = None
+ if target_name in target_dir:
+ mod = target_dir[target_name]
+
+ # Actually make the copy.
+ dup = source_obj.clone(target_dir)
target_dir._items[target_name] = dup
+ target_dir._modified = True
- self.notify(target_dir, ADD, target_name, dup)
-
+ if mod:
+ self.notify(MOD, target_dir, target_name, (mod, dup))
+ else:
+ self.notify(ADD, target_dir, target_name, dup)
@_synchronized
def manifest_text(self, strip=False, normalize=False):
@@ -951,23 +967,25 @@ class SynchronizedCollectionBase(CollectionBase):
return self._manifest_text
@_synchronized
- def diff(self, end_collection, prefix="."):
+ def diff(self, end_collection, prefix=".", holding_collection=None):
"""
Generate list of add/modify/delete actions which, when given to `apply`, will
change `self` to match `end_collection`
"""
changes = []
+ if holding_collection is None:
+ holding_collection = Collection()
for k in self:
if k not in end_collection:
- changes.append((DEL, os.path.join(prefix, k), self[k]))
+ changes.append((DEL, os.path.join(prefix, k), self[k].clone(holding_collection)))
for k in end_collection:
if k in self:
if isinstance(end_collection[k], Subcollection) and isinstance(self[k], Subcollection):
- changes.extend(self[k].diff(end_collection[k], os.path.join(prefix, k)))
+ changes.extend(self[k].diff(end_collection[k], os.path.join(prefix, k), holding_collection))
elif end_collection[k] != self[k]:
- changes.append((MOD, os.path.join(prefix, k), self[k], end_collection[k]))
+ changes.append((MOD, os.path.join(prefix, k), self[k].clone(holding_collection), end_collection[k].clone(holding_collection)))
else:
- changes.append((ADD, os.path.join(prefix, k), end_collection[k]))
+ changes.append((ADD, os.path.join(prefix, k), end_collection[k].clone(holding_collection)))
return changes
@_must_be_writable
@@ -1018,6 +1036,23 @@ class SynchronizedCollectionBase(CollectionBase):
stripped = self.manifest_text(strip=True)
return hashlib.md5(stripped).hexdigest() + '+' + str(len(stripped))
+ @_synchronized
+ def __eq__(self, other):
+ if other is self:
+ return True
+ if not isinstance(other, SynchronizedCollectionBase):
+ return False
+ if len(self._items) != len(other):
+ return False
+ for k in self._items:
+ if k not in other:
+ return False
+ if self._items[k] != other[k]:
+ return False
+ return True
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
class Collection(SynchronizedCollectionBase):
"""Store an Arvados collection, consisting of a set of files and
@@ -1305,9 +1340,9 @@ class Collection(SynchronizedCollectionBase):
self.callbacks.remove(callback)
@_synchronized
- def notify(self, collection, event, name, item):
+ def notify(self, event, collection, name, item):
for c in self.callbacks:
- c(collection, event, name, item)
+ c(event, collection, name, item)
class Subcollection(SynchronizedCollectionBase):
"""This is a subdirectory within a collection that doesn't have its own API
@@ -1335,8 +1370,8 @@ class Subcollection(SynchronizedCollectionBase):
def _populate(self):
self.parent._populate()
- def notify(self, collection, event, name, item):
- self.parent.notify(collection, event, name, item)
+ def notify(self, event, collection, name, item):
+ self.parent.notify(event, collection, name, item)
@_synchronized
def clone(self, new_parent):
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 596dd24..bff2965 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -17,6 +17,7 @@ import arvados_testutil as tutil
from arvados.ranges import Range, LocatorAndRange
from arvados import import_manifest, export_manifest
from arvados.arvfile import SYNC_EXPLICIT
+from arvados.collection import Collection
class TestResumableWriter(arvados.ResumableCollectionWriter):
KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
@@ -813,7 +814,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
"""
- self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(arvados.import_manifest(m1)))
+ self.assertEqual(". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:127:md5sum.txt\n", arvados.export_manifest(Collection(m1)))
def test_init_manifest(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
@@ -824,59 +825,59 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_remove(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
- self.assertIn("count1.txt", c)
- c.remove("count1.txt")
- self.assertNotIn("count1.txt", c)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ self.assertIn("count1.txt", c)
+ c.remove("count1.txt")
+ self.assertNotIn("count1.txt", c)
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
def test_remove_in_subdir(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- c.remove("foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c.remove("foo/count2.txt")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_remove_empty_subdir(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- c.remove("foo/count2.txt")
- c.remove("foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c.remove("foo/count2.txt")
+ c.remove("foo")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_remove_nonempty_subdir(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- with self.assertRaises(IOError):
- c.remove("foo")
- c.remove("foo", rm_r=True)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ with self.assertRaises(IOError):
+ c.remove("foo")
+ c.remove("foo", rm_r=True)
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_to_dir1(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
- c.copy("count1.txt", "foo/count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c.copy("count1.txt", "foo/count2.txt")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(c))
def test_copy_to_dir2(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
- c.copy("count1.txt", "foo")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c.copy("count1.txt", "foo")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_to_dir2(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
- c.copy("count1.txt", "foo/")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c.copy("count1.txt", "foo/")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c))
def test_copy_file(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT) as c:
- c.copy("count1.txt", "count2.txt")
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c.copy("count1.txt", "count2.txt")
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", c.manifest_text())
def test_clone(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT) as c:
- cl = c.clone()
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
+ c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ cl = c.clone()
+ self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
def test_diff1(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
('add', './count1.txt', c1["count1.txt"])])
@@ -888,8 +889,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff2(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [])
d = c1.diff(c2)
@@ -900,8 +901,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff3(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
d = c1.diff(c2)
@@ -912,8 +913,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff4(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
d = c1.diff(c2)
@@ -924,8 +925,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff5(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo', c2["foo"])])
d = c1.diff(c2)
@@ -936,8 +937,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff6(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt\n')
+
d = c2.diff(c1)
self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
('add', './foo/count2.txt', c1.find("foo/count2.txt"))])
@@ -950,8 +952,8 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
self.assertEqual(c1.manifest_text(), c2.manifest_text())
def test_diff7(self):
- c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt', sync=SYNC_EXPLICIT)
- c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo')
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo\n')
d = c2.diff(c1)
self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
d = c1.diff(c2)
@@ -961,6 +963,46 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1.apply(d)
self.assertEqual(c1.manifest_text(), c2.manifest_text())
+ def test_conflict1(self):
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt\n')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
+ ('add', './count2.txt', c2["count2.txt"])])
+ with c1.open("count1.txt", "w") as f:
+ f.write("zzzzz")
+
+ # c1 changed, so it should not be deleted.
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), ". 95ebc3c7b3b9f1d2c40fec14415d3cb8+5 5348b82a029fd9e971a811ce1f71360b+43 0:5:count1.txt 5:10:count2.txt\n")
+
+ def test_conflict2(self):
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
+ with c1.open("count1.txt", "w") as f:
+ f.write("zzzzz")
+
+ # c1 changed, so c2 mod will go to a conflict file
+ c1.apply(d)
+ self.assertTrue(re.match(r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1.txt 5:10:count1.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~$",
+ c1.manifest_text()))
+
+ def test_conflict3(self):
+ c1 = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n', sync=SYNC_EXPLICIT)
+ c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt\n')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('del', './count2.txt', c1["count2.txt"]),
+ ('add', './count1.txt', c2["count1.txt"])])
+ with c1.open("count1.txt", "w") as f:
+ f.write("zzzzz")
+
+ # c1 added count1.txt, so c2 add will go to a conflict file
+ c1.apply(d)
+ self.assertTrue(re.match(r"\. 95ebc3c7b3b9f1d2c40fec14415d3cb8\+5 5348b82a029fd9e971a811ce1f71360b\+43 0:5:count1.txt 5:10:count1.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~$",
+ c1.manifest_text()))
+
if __name__ == '__main__':
unittest.main()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list