[ARVADOS] updated: 5c0430fd80db2fd3d57243d9187588c6854a738b
git at public.curoverse.com
git at public.curoverse.com
Wed Feb 4 16:25:10 EST 2015
Summary of changes:
sdk/python/arvados/arvfile.py | 8 ++-
sdk/python/arvados/collection.py | 41 +++++++-------
sdk/python/tests/test_collections.py | 105 ++++++++++++++++++++++++++++-------
3 files changed, 113 insertions(+), 41 deletions(-)
via 5c0430fd80db2fd3d57243d9187588c6854a738b (commit)
via d42850766752bad0f1c2be55534b80617bcda411 (commit)
from c30d3a7cc1c647ed5339ac15c8d8df07a76c712d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5c0430fd80db2fd3d57243d9187588c6854a738b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Feb 4 16:27:15 2015 -0500
4823: Switched order of diff (now gives a diff *to* the target collection
instead of *from* the target collection).
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index fc8a6af..97d0605 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -608,8 +608,10 @@ class ArvadosFile(object):
return True
if type(other) != ArvadosFile:
return False
+
+ s = other.segments()
with self.lock:
- return self._segments == other.segments()
+ return self._segments == s
def __ne__(self, other):
return not self.__eq__(other)
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 0f7f2ef..3b23143 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -951,22 +951,23 @@ class SynchronizedCollectionBase(CollectionBase):
return self._manifest_text
@_synchronized
- def diff(self, start_collection, prefix="."):
+ def diff(self, end_collection, prefix="."):
"""
- Generate list of add/delete actions which change `start_collection` to result in `self`
+ Generate list of add/modify/delete actions which, when given to `apply`, will
+ change `self` to match `end_collection`
"""
changes = []
- for k in start_collection:
- if k not in self:
- changes.append((DEL, os.path.join(prefix, k), start_collection[k]))
for k in self:
- if k in start_collection:
- if isinstance(self[k], Subcollection) and isinstance(start_collection[k], Subcollection):
- changes.extend(self[k].diff(start_collection[k], os.path.join(prefix, k)))
- elif self[k] != start_collection[k]:
- changes.append((MOD, os.path.join(prefix, k), start_collection[k], self[k]))
+ if k not in end_collection:
+ changes.append((DEL, os.path.join(prefix, k), self[k]))
+ for k in end_collection:
+ if k in self:
+ if isinstance(end_collection[k], Subcollection) and isinstance(self[k], Subcollection):
+ changes.extend(self[k].diff(end_collection[k], os.path.join(prefix, k)))
+ elif end_collection[k] != self[k]:
+ changes.append((MOD, os.path.join(prefix, k), self[k], end_collection[k]))
else:
- changes.append((ADD, os.path.join(prefix, k), self[k]))
+ changes.append((ADD, os.path.join(prefix, k), end_collection[k]))
return changes
@_must_be_writable
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index aa2035a..596dd24 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -877,10 +877,10 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff1(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt')
- d = c1.diff(c2)
+ d = c2.diff(c1)
self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
('add', './count1.txt', c1["count1.txt"])])
- d = c2.diff(c1)
+ d = c1.diff(c2)
self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
('add', './count2.txt', c2["count2.txt"])])
self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
@@ -890,10 +890,10 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff2(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
- d = c1.diff(c2)
- self.assertEqual(d, [])
d = c2.diff(c1)
self.assertEqual(d, [])
+ d = c1.diff(c2)
+ self.assertEqual(d, [])
self.assertEqual(c1.manifest_text(), c2.manifest_text())
c1.apply(d)
@@ -902,9 +902,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff3(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
- d = c1.diff(c2)
- self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
d = c2.diff(c1)
+ self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
+ d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
@@ -914,9 +914,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff4(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt')
- d = c1.diff(c2)
- self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
d = c2.diff(c1)
+ self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
+ d = c1.diff(c2)
self.assertEqual(d, [('add', './count2.txt', c2["count2.txt"])])
self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
@@ -926,9 +926,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff5(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt')
- d = c1.diff(c2)
- self.assertEqual(d, [('del', './foo', c2["foo"])])
d = c2.diff(c1)
+ self.assertEqual(d, [('del', './foo', c2["foo"])])
+ d = c1.diff(c2)
self.assertEqual(d, [('add', './foo', c2["foo"])])
self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
@@ -938,10 +938,10 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff6(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt')
- d = c1.diff(c2)
+ d = c2.diff(c1)
self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
('add', './foo/count2.txt', c1.find("foo/count2.txt"))])
- d = c2.diff(c1)
+ d = c1.diff(c2)
self.assertEqual(d, [('del', './foo/count2.txt', c1.find("foo/count2.txt")),
('add', './foo/count3.txt', c2.find("foo/count3.txt"))])
@@ -952,9 +952,9 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_diff7(self):
c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt', sync=SYNC_EXPLICIT)
c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo')
- d = c1.diff(c2)
- self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
d = c2.diff(c1)
+ self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
+ d = c1.diff(c2)
self.assertEqual(d, [('mod', './foo', c1["foo"], c2["foo"])])
self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
commit d42850766752bad0f1c2be55534b80617bcda411
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Feb 4 16:04:50 2015 -0500
4823: Add tests for Collection diff and apply.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 7ad763a..fc8a6af 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -603,11 +603,13 @@ class ArvadosFile(object):
self._segments = other.segments()
self._modified = True
- @_synchronized
def __eq__(self, other):
+ if other is self:
+ return True
if type(other) != ArvadosFile:
return False
- return self._segments == other.segments()
+ with self.lock:
+ return self._segments == other.segments()
def __ne__(self, other):
return not self.__eq__(other)
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 292d421..0f7f2ef 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -783,7 +783,7 @@ class SynchronizedCollectionBase(CollectionBase):
@_synchronized
def __iter__(self):
"""Iterate over names of files and collections contained in this collection."""
- return self._items.keys()
+ return self._items.keys().__iter__()
@_synchronized
def iterkeys(self):
@@ -990,7 +990,7 @@ class SynchronizedCollectionBase(CollectionBase):
# There is already local file and it is different:
# save change to conflict file.
self.copy(initial, conflictpath)
- elif c[1] == MOD:
+ elif c[0] == MOD:
if local == initial:
# Local matches the "initial" item so assume it hasn't
# changed locally and is safe to update.
@@ -1005,8 +1005,8 @@ class SynchronizedCollectionBase(CollectionBase):
# Local is missing (presumably deleted) or local doesn't
# match the "start" value, so save change to conflict file
self.copy(c[3], conflictpath)
- elif c[1] == DEL:
- if local == initial
+ elif c[0] == DEL:
+ if local == initial:
# Local item matches "initial" value, so it is safe to remove.
self.remove(path, rm_r=True)
# else, the file is modified or already removed, in either
@@ -1073,7 +1073,6 @@ class Collection(SynchronizedCollectionBase):
self.lock = threading.RLock()
self.callbacks = []
self.events = None
- self._baseline_manifest
if manifest_locator_or_text:
if re.match(util.keep_locator_pattern, manifest_locator_or_text):
@@ -1109,10 +1108,11 @@ class Collection(SynchronizedCollectionBase):
self.update()
@_synchronized
- def update(self):
- n = self._my_api().collections().get(uuid=self._manifest_locator).execute()
- other = import_collection(n["manifest_text"])
- baseline = import_collection(self._baseline_manifest)
+ def update(self, other=None):
+ if other is None:
+ n = self._my_api().collections().get(uuid=self._manifest_locator).execute()
+ other = import_collection(n["manifest_text"])
+ baseline = import_collection(self._manifest_text)
self.apply(other.diff(baseline))
@_synchronized
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 5f86fa5..aa2035a 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -874,25 +874,92 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
cl = c.clone()
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt\n", export_manifest(cl))
- def test_merge1(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT) as c1:
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count2.txt') as c2:
- c1.merge(c2)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:10:count2.txt\n", export_manifest(c1))
-
- def test_merge2(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT) as c1:
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c2:
- c1.merge(c2)
- self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n", export_manifest(c1))
-
- def test_merge3(self):
- with arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT) as c1:
- with arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt') as c2:
- c1.merge(c2)
- self.assertTrue(re.match(r". 781e5e245d69b566979b86e28d23f2c7\+10 5348b82a029fd9e971a811ce1f71360b\+43 0:10:count1.txt 10:10:count1.txt~conflict-\d\d\d\d-\d\d-\d\d_\d\d:\d\d:\d\d~\n", export_manifest(c1)))
-
-
+ def test_diff1(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"]),
+ ('add', './count1.txt', c1["count1.txt"])])
+ d = c2.diff(c1)
+ self.assertEqual(d, [('del', './count1.txt', c1["count1.txt"]),
+ ('add', './count2.txt', c2["count2.txt"])])
+ self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+
+ def test_diff2(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [])
+ d = c2.diff(c1)
+ self.assertEqual(d, [])
+
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+
+ def test_diff3(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('mod', './count1.txt', c2["count1.txt"], c1["count1.txt"])])
+ d = c2.diff(c1)
+ self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
+
+ self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+
+ def test_diff4(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt 10:20:count2.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('del', './count2.txt', c2["count2.txt"])])
+ d = c2.diff(c1)
+ self.assertEqual(d, [('add', './count2.txt', c2["count2.txt"])])
+
+ self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+
+ def test_diff5(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('del', './foo', c2["foo"])])
+ d = c2.diff(c1)
+ self.assertEqual(d, [('add', './foo', c2["foo"])])
+
+ self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+
+ def test_diff6(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:3:count3.txt')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('del', './foo/count3.txt', c2.find("foo/count3.txt")),
+ ('add', './foo/count2.txt', c1.find("foo/count2.txt"))])
+ d = c2.diff(c1)
+ self.assertEqual(d, [('del', './foo/count2.txt', c1.find("foo/count2.txt")),
+ ('add', './foo/count3.txt', c2.find("foo/count3.txt"))])
+
+ self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
+
+ def test_diff7(self):
+ c1 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n./foo 5348b82a029fd9e971a811ce1f71360b+43 0:10:count2.txt', sync=SYNC_EXPLICIT)
+ c2 = arvados.import_manifest('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt 0:3:foo')
+ d = c1.diff(c2)
+ self.assertEqual(d, [('mod', './foo', c2["foo"], c1["foo"])])
+ d = c2.diff(c1)
+ self.assertEqual(d, [('mod', './foo', c1["foo"], c2["foo"])])
+
+ self.assertNotEqual(c1.manifest_text(), c2.manifest_text())
+ c1.apply(d)
+ self.assertEqual(c1.manifest_text(), c2.manifest_text())
if __name__ == '__main__':
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list