[ARVADOS] updated: 06d87aa4fe72ad86c94593e4909be08bad6acb35

git at public.curoverse.com git at public.curoverse.com
Tue May 12 17:13:53 EDT 2015


Summary of changes:
 sdk/python/arvados/arvfile.py          |  28 ++-
 sdk/python/arvados/collection.py       |  13 ++
 services/fuse/arvados_fuse/__init__.py |  26 +--
 services/fuse/arvados_fuse/fusedir.py  |  19 ++-
 services/fuse/tests/test_mount.py      | 299 ++++++++++++++++++++++++++++++---
 5 files changed, 349 insertions(+), 36 deletions(-)

       via  06d87aa4fe72ad86c94593e4909be08bad6acb35 (commit)
      from  682dd5b6cc23a455766a7651e3e841257660b31c (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 06d87aa4fe72ad86c94593e4909be08bad6acb35
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue May 12 17:13:47 2015 -0400

    3198: Support efficient moving preserving inode, writing to unlinked files.  Many new tests.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index c086e5f..f23fb0a 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -335,6 +335,12 @@ class _BufferBlock(object):
         bufferblock.append(self.buffer_view[0:self.size()])
         return bufferblock
 
+    @synchronized
+    def clear(self):
+        self.owner = None
+        self.buffer_block = None
+        self.buffer_view = None
+
 
 class NoopLock(object):
     def __enter__(self):
@@ -512,6 +518,12 @@ class _BlockManager(object):
     def get_bufferblock(self, locator):
         return self._bufferblocks.get(locator)
 
+    @synchronized
+    def delete_bufferblock(self, locator):
+        bb = self._bufferblocks[locator]
+        bb.clear()
+        del self._bufferblocks[locator]
+
     def get_block_contents(self, locator, num_retries, cache_only=False):
         """Fetch a block.
 
@@ -832,6 +844,19 @@ class ArvadosFile(object):
             if self._current_bblock and self._current_bblock.state() == _BufferBlock.WRITABLE:
                 self._repack_writes(num_retries)
                 self.parent._my_block_manager().commit_bufferblock(self._current_bblock, wait)
+            if wait:
+                to_delete = set()
+                for s in self._segments:
+                    bb = self.parent._my_block_manager().get_bufferblock(s.locator)
+                    if bb:
+                        if bb.state() != _BufferBlock.COMMITTED:
+                            _logger.error("bufferblock %s is not committed" % (s.locator))
+                        else:
+                            to_delete.add(s.locator)
+                            s.locator = bb.locator()
+                for s in to_delete:
+                   self.parent._my_block_manager().delete_bufferblock(s)
+
             self.parent.notify(MOD, self.parent, self.name, (self, self))
 
     @must_be_writable
@@ -881,13 +906,14 @@ class ArvadosFile(object):
     @must_be_writable
     @synchronized
     def reparent(self, newparent, newname):
+        self._modified = True
         self.flush()
         self.parent.remove(self.name)
 
         self.parent = newparent
         self.name = newname
         self.lock = self.parent.root_collection().lock
-        self._modified = True
+
 
 class ArvadosFileReader(ArvadosFileReaderBase):
     """Wraps ArvadosFile in a file-like object supporting reading only.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 89cbda9..eb1449b 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -988,6 +988,8 @@ class RichCollectionBase(CollectionBase):
         alternate path indicating the conflict.
 
         """
+        if changes:
+            self._modified = True
         for change in changes:
             event_type = change[0]
             path = change[1]
@@ -1190,6 +1192,7 @@ class Collection(RichCollectionBase):
             other = CollectionReader(response["manifest_text"])
         baseline = CollectionReader(self._manifest_text)
         self.apply(baseline.diff(other))
+        self._manifest_text = self.manifest_text()
 
     @synchronized
     def _my_api(self):
@@ -1555,6 +1558,16 @@ class Subcollection(RichCollectionBase):
         c._clonefrom(self)
         return c
 
+    @must_be_writable
+    @synchronized
+    def reparent(self, newparent, newname):
+        # XXX add flush()
+        self.parent.remove(self.name, recursive=True)
+        self.parent = newparent
+        self.name = newname
+        self.lock = self.parent.root_collection().lock
+        self._modified = True
+
 
 class CollectionReader(Collection):
     """A read-only collection object.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b85e65d..d5350f4 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -70,7 +70,7 @@ class InodeCache(object):
     def __init__(self, cap, min_entries=4):
         self._entries = collections.OrderedDict()
         self._by_uuid = {}
-        self._counter = itertools.count(1)
+        self._counter = itertools.count(0)
         self.cap = cap
         self._total = 0
         self.min_entries = min_entries
@@ -156,19 +156,22 @@ class Inodes(object):
 
     def add_entry(self, entry):
         entry.inode = next(self._counter)
+        if entry.inode == llfuse.ROOT_INODE:
+            entry.inc_ref()
         self._entries[entry.inode] = entry
         self.inode_cache.manage(entry)
         return entry
 
     def del_entry(self, entry):
         if entry.ref_count == 0:
-            _logger.warn("Deleting inode %i", entry.inode)
+            _logger.debug("Deleting inode %i", entry.inode)
             self.inode_cache.unmanage(entry)
             llfuse.invalidate_inode(entry.inode)
             del self._entries[entry.inode]
+            entry.inode = None
         else:
-            _logger.warn("Inode %i has refcount %i", entry.inode, entry.ref_count)
             entry.dead = True
+            _logger.debug("del_entry on inode %i with refcount %i", entry.inode, entry.ref_count)
 
 def catch_exceptions(orig_func):
     @functools.wraps(orig_func)
@@ -210,7 +213,7 @@ class Operations(llfuse.Operations):
 
         # dict of inode to filehandle
         self._filehandles = {}
-        self._filehandles_counter = 1
+        self._filehandles_counter = itertools.count(0)
 
         # Other threads that need to wait until the fuse driver
         # is fully initialized should wait() on this event object.
@@ -317,8 +320,8 @@ class Operations(llfuse.Operations):
     @catch_exceptions
     def forget(self, inodes):
         for inode, nlookup in inodes:
-            _logger.debug("arv-mount forget: %i %i", inode, nlookup)
             ent = self.inodes[inode]
+            _logger.debug("arv-mount forget: inode %i nlookup %i ref_count %i", inode, nlookup, ent.ref_count)
             if ent.dec_ref(nlookup) == 0 and ent.dead:
                 self.inodes.del_entry(ent)
 
@@ -335,8 +338,7 @@ class Operations(llfuse.Operations):
         if ((flags & os.O_WRONLY) or (flags & os.O_RDWR)) and not p.writable():
             raise llfuse.FUSEError(errno.EPERM)
 
-        fh = self._filehandles_counter
-        self._filehandles_counter += 1
+        fh = next(self._filehandles_counter)
         self._filehandles[fh] = FileHandle(fh, p)
         self.inodes.touch(p)
         return fh
@@ -402,8 +404,7 @@ class Operations(llfuse.Operations):
         if not isinstance(p, Directory):
             raise llfuse.FUSEError(errno.ENOTDIR)
 
-        fh = self._filehandles_counter
-        self._filehandles_counter += 1
+        fh = next(self._filehandles_counter)
         if p.parent_inode in self.inodes:
             parent = self.inodes[p.parent_inode]
         else:
@@ -477,8 +478,7 @@ class Operations(llfuse.Operations):
 
         # The file entry should have been implicitly created by callback.
         f = p[name]
-        fh = self._filehandles_counter
-        self._filehandles_counter += 1
+        fh = next(self._filehandles_counter)
         self._filehandles[fh] = FileHandle(fh, f)
         self.inodes.touch(p)
 
@@ -487,6 +487,8 @@ class Operations(llfuse.Operations):
 
     @catch_exceptions
     def mkdir(self, inode_parent, name, mode, ctx):
+        _logger.debug("arv-mount mkdir: %i '%s' %o", inode_parent, name, mode)
+
         p = self._check_writable(inode_parent)
 
         with llfuse.lock_released:
@@ -500,6 +502,7 @@ class Operations(llfuse.Operations):
 
     @catch_exceptions
     def unlink(self, inode_parent, name):
+        _logger.debug("arv-mount unlink: %i '%s'", inode_parent, name)
         p = self._check_writable(inode_parent)
 
         with llfuse.lock_released:
@@ -510,6 +513,7 @@ class Operations(llfuse.Operations):
 
     @catch_exceptions
     def rename(self, inode_parent_old, name_old, inode_parent_new, name_new):
+        _logger.debug("arv-mount rename: %i '%s' %i '%s'", inode_parent_old, name_old, inode_parent_new, name_new)
         src = self._check_writable(inode_parent_old)
         dest = self._check_writable(inode_parent_new)
 
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 0e86c8d..16e5325 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -178,15 +178,23 @@ class CollectionDirectoryBase(Directory):
 
     def new_entry(self, name, item, mtime):
         name = sanitize_filename(name)
-        if isinstance(item, arvados.collection.RichCollectionBase):
+        if hasattr(item, "fuse_entry") and item.fuse_entry is not None:
+            if item.fuse_entry.dead is not True:
+                raise Exception("Can only reparent dead inode entry")
+            if item.fuse_entry.inode is None:
+                raise Exception("Reparented entry must still have valid inode")
+            item.fuse_entry.dead = False
+            self._entries[name] = item.fuse_entry
+        elif isinstance(item, arvados.collection.RichCollectionBase):
             self._entries[name] = self.inodes.add_entry(CollectionDirectoryBase(self.inode, self.inodes, item))
             self._entries[name].populate(mtime)
         else:
             self._entries[name] = self.inodes.add_entry(FuseArvadosFile(self.inode, item, mtime))
+        item.fuse_entry = self._entries[name]
 
     def on_event(self, event, collection, name, item):
-        _logger.warn("Got event! %s %s %s %s", event, collection, name, item)
         if collection == self.collection:
+            _logger.debug("%s %s %s %s", event, collection, name, item)
             with llfuse.lock:
                 if event == arvados.collection.ADD:
                     self.new_entry(name, item, self.mtime())
@@ -196,9 +204,10 @@ class CollectionDirectoryBase(Directory):
                     llfuse.invalidate_entry(self.inode, name)
                     self.inodes.del_entry(ent)
                 elif event == arvados.collection.MOD:
-                    ent = self._entries[name]
-                    llfuse.invalidate_inode(ent.inode)
-        _logger.warn("Finished handling event")
+                    if hasattr(item, "fuse_entry") and item.fuse_entry is not None:
+                        llfuse.invalidate_inode(item.fuse_entry.inode)
+                    elif name in self._entries:
+                        llfuse.invalidate_inode(self._entries[name].inode)
 
     def populate(self, mtime):
         self._mtime = mtime
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index c7d6387..7bc56b3 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -414,8 +414,11 @@ class FuseCreateFileTest(MountTestBase):
             r'\. d41d8cd98f00b204e9800998ecf8427e\+0\+A[a-f0-9]{40}@[a-f0-9]{8} 0:0:file1\.txt$')
 
 def fuseWriteFileTestHelper(mounttmp):
-    with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
-        return f.read() == "Hello world!"
+    class Test(unittest.TestCase):
+        def runTest(self):
+            with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
+                self.assertEqual(f.read(), "Hello world!")
+    Test().runTest()
 
 class FuseWriteFileTest(MountTestBase):
     def runTest(self):
@@ -445,7 +448,7 @@ class FuseWriteFileTest(MountTestBase):
         # workaround is to run some of our test code in a separate process.
         # Forturnately the multiprocessing module makes this relatively easy.
         pool = multiprocessing.Pool(1)
-        self.assertTrue(pool.apply(fuseWriteFileTestHelper, (self.mounttmp,)))
+        pool.apply(fuseWriteFileTestHelper, (self.mounttmp,))
         pool.close()
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -453,21 +456,25 @@ class FuseWriteFileTest(MountTestBase):
             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 
 def fuseUpdateFileTestHelper1(mounttmp):
-    with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
-        fr = f.read()
-        if fr != "Hello world!":
-            raise Exception("Got %s expected 'Hello world!'" % fr)
-        f.seek(0)
-        f.write("Hola mundo!")
-        f.seek(0)
-        fr = f.read()
-        if fr != "Hola mundo!!":
-            raise Exception("Got %s expected 'Hola mundo!!'" % fr)
-        return True
+    class Test(unittest.TestCase):
+        def runTest(self):
+            with open(os.path.join(mounttmp, "file1.txt"), "r+") as f:
+                fr = f.read()
+                self.assertEqual(fr, "Hello world!")
+                f.seek(0)
+                f.write("Hola mundo!")
+                f.seek(0)
+                fr = f.read()
+                self.assertEqual(fr, "Hola mundo!!")
+                return True
+    Test().runTest()
 
 def fuseUpdateFileTestHelper2(mounttmp):
-    with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
-        return f.read() == "Hola mundo!!"
+    class Test(unittest.TestCase):
+        def runTest(self):
+            with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
+                return f.read() == "Hola mundo!!"
+    Test().runTest()
 
 class FuseUpdateFileTest(MountTestBase):
     def runTest(self):
@@ -486,8 +493,8 @@ class FuseUpdateFileTest(MountTestBase):
 
         # See note in FuseWriteFileTest
         pool = multiprocessing.Pool(1)
-        self.assertTrue(pool.apply(fuseUpdateFileTestHelper1, (self.mounttmp,)))
-        self.assertTrue(pool.apply(fuseUpdateFileTestHelper2, (self.mounttmp,)))
+        pool.apply(fuseUpdateFileTestHelper1, (self.mounttmp,))
+        pool.apply(fuseUpdateFileTestHelper2, (self.mounttmp,))
         pool.close()
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -590,7 +597,7 @@ class FuseRmTest(MountTestBase):
         self.assertEqual(collection2["manifest_text"], "")
 
 
-class FuseMvTest(MountTestBase):
+class FuseMvFileTest(MountTestBase):
     def runTest(self):
         arvados.logger.setLevel(logging.DEBUG)
 
@@ -629,6 +636,45 @@ class FuseMvTest(MountTestBase):
             r'\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
 
 
+class FuseRenameTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(collection.api_response(), collection)
+        self.assertTrue(m.writable())
+
+        os.mkdir(os.path.join(self.mounttmp, "testdir"))
+
+        with open(os.path.join(self.mounttmp, "testdir", "file1.txt"), "w") as f:
+            f.write("Hello world!")
+
+        # Starting manifest
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        self.assertEqual(["testdir"], d1)
+        d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
+        self.assertEqual(["file1.txt"], d1)
+
+        os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
+
+        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        self.assertEqual(["testdir2"], sorted(d1))
+        d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
+        self.assertEqual(["file1.txt"], d1)
+
+        collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
+        self.assertRegexpMatches(collection2["manifest_text"],
+            r'\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$')
+
+
 class FuseUpdateFromEventTest(MountTestBase):
     def runTest(self):
         arvados.logger.setLevel(logging.DEBUG)
@@ -657,6 +703,221 @@ class FuseUpdateFromEventTest(MountTestBase):
         self.assertEqual(["file1.txt"], sorted(d1))
 
 
+def fuseFileConflictTestHelper(mounttmp):
+    class Test(unittest.TestCase):
+        def runTest(self):
+            d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
+            self.assertEqual(len(d1), 2)
+
+            with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
+                self.assertEqual(f.read(), "bar")
+
+            self.assertRegexpMatches(d1[1],
+                r'file1\.txt~conflict-\d\d\d\d-\d\d-\d\d-\d\d:\d\d:\d\d~')
+
+            with open(os.path.join(mounttmp, d1[1]), "r") as f:
+                self.assertEqual(f.read(), "foo")
+
+    Test().runTest()
+
+class FuseFileConflictTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(collection.api_response(), collection)
+
+        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        self.assertEqual([], sorted(d1))
+
+        with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
+            with collection2.open("file1.txt", "w") as f:
+                f.write("foo")
+
+        with open(os.path.join(self.mounttmp, "file1.txt"), "w") as f:
+            f.write("bar")
+
+        # See comment in FuseWriteFileTest
+        pool = multiprocessing.Pool(1)
+        pool.apply(fuseFileConflictTestHelper, (self.mounttmp,))
+        pool.close()
+
+
+def fuseUnlinkOpenFileTest(mounttmp):
+    class Test(unittest.TestCase):
+        def runTest(self):
+            with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
+                f.write("foo")
+
+                d1 = llfuse.listdir(os.path.join(mounttmp))
+                self.assertEqual(["file1.txt"], sorted(d1))
+
+                os.remove(os.path.join(mounttmp, "file1.txt"))
+
+                d1 = llfuse.listdir(os.path.join(mounttmp))
+                self.assertEqual([], sorted(d1))
+
+                f.seek(0)
+                self.assertEqual(f.read(), "foo")
+                f.write("bar")
+
+                f.seek(0)
+                self.assertEqual(f.read(), "foobar")
+
+    Test().runTest()
+
+class FuseUnlinkOpenFileTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection = arvados.collection.Collection(api_client=self.api)
+        collection.save_new()
+
+        m = self.make_mount(fuse.CollectionDirectory)
+        with llfuse.lock:
+            m.new_collection(collection.api_response(), collection)
+
+        # See comment in FuseWriteFileTest
+        pool = multiprocessing.Pool(1)
+        pool.apply(fuseUnlinkOpenFileTest, (self.mounttmp,))
+        pool.close()
+
+        self.assertEqual(collection.manifest_text(), "")
+
+
+def fuseMvFileBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
+    class Test(unittest.TestCase):
+        def runTest(self):
+            with open(os.path.join(mounttmp, uuid1, "file1.txt"), "w") as f:
+                f.write("Hello world!")
+
+            d1 = os.listdir(os.path.join(mounttmp, uuid1))
+            self.assertEqual(["file1.txt"], sorted(d1))
+            d1 = os.listdir(os.path.join(mounttmp, uuid2))
+            self.assertEqual([], sorted(d1))
+
+    Test().runTest()
+
+def fuseMvFileBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
+    class Test(unittest.TestCase):
+        def runTest(self):
+            os.rename(os.path.join(mounttmp, uuid1, "file1.txt"), os.path.join(mounttmp, uuid2, "file2.txt"))
+
+            d1 = os.listdir(os.path.join(mounttmp, uuid1))
+            self.assertEqual([], sorted(d1))
+            d1 = os.listdir(os.path.join(mounttmp, uuid2))
+            self.assertEqual(["file2.txt"], sorted(d1))
+
+    Test().runTest()
+
+class FuseMvFileBetweenCollectionsTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection1 = arvados.collection.Collection(api_client=self.api)
+        collection1.save_new()
+
+        collection2 = arvados.collection.Collection(api_client=self.api)
+        collection2.save_new()
+
+        m = self.make_mount(fuse.MagicDirectory)
+
+        # See comment in FuseWriteFileTest
+        pool = multiprocessing.Pool(1)
+        pool.apply(fuseMvFileBetweenCollectionsTest1, (self.mounttmp,
+                                                  collection1.manifest_locator(),
+                                                  collection2.manifest_locator()))
+
+        collection1.update()
+        collection2.update()
+
+        self.assertRegexpMatches(collection1.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+        self.assertEqual(collection2.manifest_text(), "")
+
+        pool.apply(fuseMvFileBetweenCollectionsTest2, (self.mounttmp,
+                                                  collection1.manifest_locator(),
+                                                  collection2.manifest_locator()))
+        pool.close()
+
+        collection1.update()
+        collection2.update()
+
+        self.assertEqual(collection1.manifest_text(), "")
+        self.assertRegexpMatches(collection2.manifest_text(), r"\. 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file2\.txt$")
+
+
+def fuseMvDirBetweenCollectionsTest1(mounttmp, uuid1, uuid2):
+    class Test(unittest.TestCase):
+        def runTest(self):
+            os.mkdir(os.path.join(mounttmp, uuid1, "testdir"))
+            with open(os.path.join(mounttmp, uuid1, "testdir", "file1.txt"), "w") as f:
+                f.write("Hello world!")
+
+            d1 = os.listdir(os.path.join(mounttmp, uuid1))
+            self.assertEqual(["testdir"], sorted(d1))
+            d1 = os.listdir(os.path.join(mounttmp, uuid1, "testdir"))
+            self.assertEqual(["file1.txt"], sorted(d1))
+
+            d1 = os.listdir(os.path.join(mounttmp, uuid2))
+            self.assertEqual([], sorted(d1))
+
+    Test().runTest()
+
+def fuseMvDirBetweenCollectionsTest2(mounttmp, uuid1, uuid2):
+    class Test(unittest.TestCase):
+        def runTest(self):
+            os.rename(os.path.join(mounttmp, uuid1, "testdir"), os.path.join(mounttmp, uuid2, "testdir2"))
+
+            d1 = os.listdir(os.path.join(mounttmp, uuid1))
+            self.assertEqual([], sorted(d1))
+
+            d1 = os.listdir(os.path.join(mounttmp, uuid2))
+            self.assertEqual(["testdir2"], sorted(d1))
+            d1 = os.listdir(os.path.join(mounttmp, uuid2, "testdir2"))
+            self.assertEqual(["file1.txt"], sorted(d1))
+
+    Test().runTest()
+
+class FuseMvDirBetweenCollectionsTest(MountTestBase):
+    def runTest(self):
+        arvados.logger.setLevel(logging.DEBUG)
+
+        collection1 = arvados.collection.Collection(api_client=self.api)
+        collection1.save_new()
+
+        collection2 = arvados.collection.Collection(api_client=self.api)
+        collection2.save_new()
+
+        m = self.make_mount(fuse.MagicDirectory)
+
+        # See comment in FuseWriteFileTest
+        pool = multiprocessing.Pool(1)
+        pool.apply(fuseMvDirBetweenCollectionsTest1, (self.mounttmp,
+                                                  collection1.manifest_locator(),
+                                                  collection2.manifest_locator()))
+
+        collection1.update()
+        collection2.update()
+
+        self.assertRegexpMatches(collection1.manifest_text(), r"\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+        self.assertEqual(collection2.manifest_text(), "")
+
+        pool.apply(fuseMvDirBetweenCollectionsTest2, (self.mounttmp,
+                                                  collection1.manifest_locator(),
+                                                  collection2.manifest_locator()))
+        pool.close()
+
+        collection1.update()
+        collection2.update()
+
+        self.assertEqual(collection1.manifest_text(), "")
+        self.assertRegexpMatches(collection2.manifest_text(), r"\./testdir2 86fb269d190d2c85f6e0468ceca42a20\+12\+A[a-f0-9]{40}@[a-f0-9]{8} 0:12:file1\.txt$")
+
+
 class FuseUnitTest(unittest.TestCase):
     def test_sanitize_filename(self):
         acceptable = [

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list