[ARVADOS] updated: 503c686bc80825d00980a970af69ec60f9e6ce9b

Git user git at public.curoverse.com
Wed Sep 14 12:06:20 EDT 2016


Summary of changes:
 services/fuse/arvados_fuse/command.py   |  1 +
 services/fuse/tests/integration_test.py | 51 +++++++++++++++++++++++----------
 services/fuse/tests/mount_test_base.py  | 26 ++++++++++-------
 3 files changed, 52 insertions(+), 26 deletions(-)

       via  503c686bc80825d00980a970af69ec60f9e6ce9b (commit)
       via  2dc308e3106090e2a19185e26cff22c6136bb5c4 (commit)
       via  1c582d3ee6df9e3ad114ec3ec0cbc58f52626ffb (commit)
       via  7382b7a2d35e93944ff364d36017f51e9a1cd392 (commit)
       via  caa4cce0bc97ae99bc1657c3126d3c098e959eaa (commit)
       via  89c7264f5a3b53043762fb8ced7bdb997ed2a120 (commit)
      from  849f35d1cafa1b58e55832426436689b95d69ac5 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 503c686bc80825d00980a970af69ec60f9e6ce9b
Merge: 849f35d 2dc308e
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Sep 14 12:04:00 2016 -0400

    Merge branch '9986-fuse-test-deadlock' closes #9986


commit 2dc308e3106090e2a19185e26cff22c6136bb5c4
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Sep 14 11:04:35 2016 -0400

    9986: Move llfuse thread-shutdown check into a "finally" block so it actually runs.

diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index c4b0df3..d15f017 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -117,6 +117,7 @@ class Mount(object):
         self.llfuse_thread.daemon = True
         self.llfuse_thread.start()
         self.operations.initlock.wait()
+        return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index c705934..b485037 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -4,6 +4,7 @@ import arvados_fuse.command
 import atexit
 import functools
 import inspect
+import logging
 import multiprocessing
 import os
 import run_test_server
@@ -72,16 +73,19 @@ class IntegrationTest(unittest.TestCase):
         def decorator(func):
             @functools.wraps(func)
             def wrapper(self, *args, **kwargs):
-                with arvados_fuse.command.Mount(
-                        arvados_fuse.command.ArgumentParser().parse_args(
-                            argv + ['--foreground',
-                                    '--unmount-timeout=0.1',
-                                    self.mnt])) as m:
-                    return func(self, *args, **kwargs)
-                if m.llfuse_thread.is_alive():
-                    self.logger.warning("IntegrationTest.mount:"
-                                        " llfuse thread still alive after umount"
-                                        " -- killing test suite to avoid deadlock")
-                    os.kill(os.getpid(), signal.SIGKILL)
+                self.mount = None
+                try:
+                    with arvados_fuse.command.Mount(
+                            arvados_fuse.command.ArgumentParser().parse_args(
+                                argv + ['--foreground',
+                                        '--unmount-timeout=2',
+                                        self.mnt])) as self.mount:
+                        return func(self, *args, **kwargs)
+                finally:
+                    if self.mount and self.mount.llfuse_thread.is_alive():
+                        logging.warning("IntegrationTest.mount:"
+                                            " llfuse thread still alive after umount"
+                                            " -- killing test suite to avoid deadlock")
+                        os.kill(os.getpid(), signal.SIGKILL)
             return wrapper
         return decorator

commit 1c582d3ee6df9e3ad114ec3ec0cbc58f52626ffb
Author: Tom Clegg <tom at curoverse.com>
Date:   Wed Sep 14 11:03:43 2016 -0400

    9986: Report actual thread shutdown wait if >0.1s (instead of just saying it took >1s)

diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 5e0e4c7..20192f9 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -67,17 +67,16 @@ class MountTestBase(unittest.TestCase):
     def tearDown(self):
         if self.llfuse_thread:
             subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
-            self.llfuse_thread.join(timeout=1)
-            if self.llfuse_thread.is_alive():
-                logger.warning("MountTestBase.tearDown():"
-                               " llfuse thread still alive 1s after umount"
-                               " -- waiting another 10s")
-                self.llfuse_thread.join(timeout=10)
+            t0 = time.time()
+            self.llfuse_thread.join(timeout=10)
             if self.llfuse_thread.is_alive():
                 logger.warning("MountTestBase.tearDown():"
                                " llfuse thread still alive 10s after umount"
                                " -- exiting with SIGKILL")
                 os.kill(os.getpid(), signal.SIGKILL)
+            waited = time.time() - t0
+            if waited > 0.1:
+                logger.warning("MountTestBase.tearDown(): waited %f s for llfuse thread to end", waited)
 
         os.rmdir(self.mounttmp)
         if self.keeptmp:

commit 7382b7a2d35e93944ff364d36017f51e9a1cd392
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 8 14:26:58 2016 -0400

    9986: Kill test suite when danger of deadlock is detected.

diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index d0e3102..c705934 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -76,7 +76,12 @@ class IntegrationTest(unittest.TestCase):
                         arvados_fuse.command.ArgumentParser().parse_args(
                             argv + ['--foreground',
                                     '--unmount-timeout=0.1',
-                                    self.mnt])):
+                                    self.mnt])) as m:
                     return func(self, *args, **kwargs)
+                if m.llfuse_thread.is_alive():
+                    self.logger.warning("IntegrationTest.mount:"
+                                        " llfuse thread still alive after umount"
+                                        " -- killing test suite to avoid deadlock")
+                    os.kill(os.getpid(), signal.SIGKILL)
             return wrapper
         return decorator

commit caa4cce0bc97ae99bc1657c3126d3c098e959eaa
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 8 09:34:31 2016 -0400

    9986: Share one multiprocessing pool with all IntegrationTests.

diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 5a45bfc..d0e3102 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -1,14 +1,28 @@
 import arvados
 import arvados_fuse
 import arvados_fuse.command
+import atexit
 import functools
 import inspect
 import multiprocessing
 import os
+import run_test_server
+import signal
 import sys
 import tempfile
 import unittest
-import run_test_server
+
+_pool = None
+
+
+ at atexit.register
+def _pool_cleanup():
+    global _pool
+    if _pool is None:
+        return
+    _pool.close()
+    _pool.join()
+
 
 def wrap_static_test_method(modName, clsName, funcName, args, kwargs):
     class Test(unittest.TestCase):
@@ -24,17 +38,15 @@ class IntegrationTest(unittest.TestCase):
         If called by method 'foobar', the static method '_foobar' of
         the same class will be called in the other process.
         """
+        global _pool
+        if _pool is None:
+            _pool = multiprocessing.Pool(1, maxtasksperchild=1)
         modName = inspect.getmodule(self).__name__
         clsName = self.__class__.__name__
         funcName = inspect.currentframe().f_back.f_code.co_name
-        pool = multiprocessing.Pool(1)
-        try:
-            pool.apply(
-                wrap_static_test_method,
-                (modName, clsName, '_'+funcName, args, kwargs))
-        finally:
-            pool.terminate()
-            pool.join()
+        _pool.apply(
+            wrap_static_test_method,
+            (modName, clsName, '_'+funcName, args, kwargs))
 
     @classmethod
     def setUpClass(cls):

commit 89c7264f5a3b53043762fb8ced7bdb997ed2a120
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 8 08:56:18 2016 -0400

    9986: Send SIGKILL if llfuse thread does not join.

diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 12395d7..5e0e4c7 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -1,18 +1,19 @@
 import arvados
-import arvados.safeapi
 import arvados_fuse as fuse
+import arvados.safeapi
 import llfuse
+import logging
+import multiprocessing
 import os
+import run_test_server
 import shutil
+import signal
 import subprocess
 import sys
 import tempfile
 import threading
 import time
 import unittest
-import logging
-import multiprocessing
-import run_test_server
 
 logger = logging.getLogger('arvados.arv-mount')
 
@@ -64,23 +65,27 @@ class MountTestBase(unittest.TestCase):
         return self.operations.inodes[llfuse.ROOT_INODE]
 
     def tearDown(self):
-        self.pool.terminate()
-        self.pool.join()
-        del self.pool
-
         if self.llfuse_thread:
             subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
             self.llfuse_thread.join(timeout=1)
             if self.llfuse_thread.is_alive():
                 logger.warning("MountTestBase.tearDown():"
                                " llfuse thread still alive 1s after umount"
-                               " -- abandoning and exiting anyway")
+                               " -- waiting another 10s")
+                self.llfuse_thread.join(timeout=10)
+            if self.llfuse_thread.is_alive():
+                logger.warning("MountTestBase.tearDown():"
+                               " llfuse thread still alive 10s after umount"
+                               " -- exiting with SIGKILL")
+                os.kill(os.getpid(), signal.SIGKILL)
 
         os.rmdir(self.mounttmp)
         if self.keeptmp:
             shutil.rmtree(self.keeptmp)
             os.environ.pop('KEEP_LOCAL_STORE')
         run_test_server.reset()
+        self.pool.close()
+        self.pool.join()
 
     def assertDirContents(self, subdir, expect_content):
         path = self.mounttmp

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list