[ARVADOS] created: a43f24b3e65d1aa93621d61f7ccd5fddf762af12

[Git user]

        at  a43f24b3e65d1aa93621d61f7ccd5fddf762af12 (commit)


commit a43f24b3e65d1aa93621d61f7ccd5fddf762af12
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 8 09:47:18 2016 -0400

    8345: Kill test suite when danger of deadlock is detected.

diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 5b11d23..fba4560 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -72,7 +72,12 @@ class IntegrationTest(unittest.TestCase):
                         arvados_fuse.command.ArgumentParser().parse_args(
                             argv + ['--foreground',
                                     '--unmount-timeout=0.1',
-                                    self.mnt])):
+                                    self.mnt])) as m:
                     return func(self, *args, **kwargs)
+                if m.llfuse_thread.is_alive():
+                    self.logger.warning("IntegrationTest.mount:"
+                                        " llfuse thread still alive after umount"
+                                        " -- killing test suite to avoid deadlock")
+                    os.kill(os.getpid(), signal.SIGKILL)
             return wrapper
         return decorator

commit 1a8526e424e1491b9f262fab7910aeca3b223d59
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 8 09:34:31 2016 -0400

    9986: Share one multiprocessing pool with all IntegrationTests.

diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 5a45bfc..5b11d23 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -1,14 +1,24 @@
 import arvados
 import arvados_fuse
 import arvados_fuse.command
+import atexit
 import functools
 import inspect
 import multiprocessing
 import os
+import run_test_server
+import signal
 import sys
 import tempfile
 import unittest
-import run_test_server
+
+_pool = None
+def _pool_cleanup():
+    if _pool is None:
+        return
+    _pool.close()
+    _pool.join()
+atexit.register(_pool_cleanup)
 
 def wrap_static_test_method(modName, clsName, funcName, args, kwargs):
     class Test(unittest.TestCase):
@@ -24,17 +34,15 @@ class IntegrationTest(unittest.TestCase):
         If called by method 'foobar', the static method '_foobar' of
         the same class will be called in the other process.
         """
+        global _pool
+        if _pool is None:
+            _pool = multiprocessing.Pool(1, maxtasksperchild=1)
         modName = inspect.getmodule(self).__name__
         clsName = self.__class__.__name__
         funcName = inspect.currentframe().f_back.f_code.co_name
-        pool = multiprocessing.Pool(1)
-        try:
-            pool.apply(
-                wrap_static_test_method,
-                (modName, clsName, '_'+funcName, args, kwargs))
-        finally:
-            pool.terminate()
-            pool.join()
+        _pool.apply(
+            wrap_static_test_method,
+            (modName, clsName, '_'+funcName, args, kwargs))
 
     @classmethod
     def setUpClass(cls):

commit 67a83b5152b72548f0e55b137f6c789f4b6b086f
Author: Tom Clegg <tom at curoverse.com>
Date:   Thu Sep 8 08:56:18 2016 -0400

    9986: Send SIGKILL if llfuse thread does not join.

diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 12395d7..5e0e4c7 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -1,18 +1,19 @@
 import arvados
-import arvados.safeapi
 import arvados_fuse as fuse
+import arvados.safeapi
 import llfuse
+import logging
+import multiprocessing
 import os
+import run_test_server
 import shutil
+import signal
 import subprocess
 import sys
 import tempfile
 import threading
 import time
 import unittest
-import logging
-import multiprocessing
-import run_test_server
 
 logger = logging.getLogger('arvados.arv-mount')
 
@@ -64,23 +65,27 @@ class MountTestBase(unittest.TestCase):
         return self.operations.inodes[llfuse.ROOT_INODE]
 
     def tearDown(self):
-        self.pool.terminate()
-        self.pool.join()
-        del self.pool
-
         if self.llfuse_thread:
             subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
             self.llfuse_thread.join(timeout=1)
             if self.llfuse_thread.is_alive():
                 logger.warning("MountTestBase.tearDown():"
                                " llfuse thread still alive 1s after umount"
-                               " -- abandoning and exiting anyway")
+                               " -- waiting another 10s")
+                self.llfuse_thread.join(timeout=10)
+            if self.llfuse_thread.is_alive():
+                logger.warning("MountTestBase.tearDown():"
+                               " llfuse thread still alive 10s after umount"
+                               " -- exiting with SIGKILL")
+                os.kill(os.getpid(), signal.SIGKILL)
 
         os.rmdir(self.mounttmp)
         if self.keeptmp:
             shutil.rmtree(self.keeptmp)
             os.environ.pop('KEEP_LOCAL_STORE')
         run_test_server.reset()
+        self.pool.close()
+        self.pool.join()
 
     def assertDirContents(self, subdir, expect_content):
         path = self.mounttmp

-----------------------------------------------------------------------


hooks/post-receive
--