[ARVADOS] updated: 503c686bc80825d00980a970af69ec60f9e6ce9b
Git user
git at public.curoverse.com
Wed Sep 14 12:06:20 EDT 2016
Summary of changes:
services/fuse/arvados_fuse/command.py | 1 +
services/fuse/tests/integration_test.py | 51 +++++++++++++++++++++++----------
services/fuse/tests/mount_test_base.py | 26 ++++++++++-------
3 files changed, 52 insertions(+), 26 deletions(-)
via 503c686bc80825d00980a970af69ec60f9e6ce9b (commit)
via 2dc308e3106090e2a19185e26cff22c6136bb5c4 (commit)
via 1c582d3ee6df9e3ad114ec3ec0cbc58f52626ffb (commit)
via 7382b7a2d35e93944ff364d36017f51e9a1cd392 (commit)
via caa4cce0bc97ae99bc1657c3126d3c098e959eaa (commit)
via 89c7264f5a3b53043762fb8ced7bdb997ed2a120 (commit)
from 849f35d1cafa1b58e55832426436689b95d69ac5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 503c686bc80825d00980a970af69ec60f9e6ce9b
Merge: 849f35d 2dc308e
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 14 12:04:00 2016 -0400
Merge branch '9986-fuse-test-deadlock' closes #9986
commit 2dc308e3106090e2a19185e26cff22c6136bb5c4
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 14 11:04:35 2016 -0400
9986: Move llfuse thread-shutdown check into a "finally" block so it actually runs.
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index c4b0df3..d15f017 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -117,6 +117,7 @@ class Mount(object):
self.llfuse_thread.daemon = True
self.llfuse_thread.start()
self.operations.initlock.wait()
+ return self
def __exit__(self, exc_type, exc_value, traceback):
subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index c705934..b485037 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -4,6 +4,7 @@ import arvados_fuse.command
import atexit
import functools
import inspect
+import logging
import multiprocessing
import os
import run_test_server
@@ -72,16 +73,19 @@ class IntegrationTest(unittest.TestCase):
def decorator(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
- with arvados_fuse.command.Mount(
- arvados_fuse.command.ArgumentParser().parse_args(
- argv + ['--foreground',
- '--unmount-timeout=0.1',
- self.mnt])) as m:
- return func(self, *args, **kwargs)
- if m.llfuse_thread.is_alive():
- self.logger.warning("IntegrationTest.mount:"
- " llfuse thread still alive after umount"
- " -- killing test suite to avoid deadlock")
- os.kill(os.getpid(), signal.SIGKILL)
+ self.mount = None
+ try:
+ with arvados_fuse.command.Mount(
+ arvados_fuse.command.ArgumentParser().parse_args(
+ argv + ['--foreground',
+ '--unmount-timeout=2',
+ self.mnt])) as self.mount:
+ return func(self, *args, **kwargs)
+ finally:
+ if self.mount and self.mount.llfuse_thread.is_alive():
+ logging.warning("IntegrationTest.mount:"
+ " llfuse thread still alive after umount"
+ " -- killing test suite to avoid deadlock")
+ os.kill(os.getpid(), signal.SIGKILL)
return wrapper
return decorator
commit 1c582d3ee6df9e3ad114ec3ec0cbc58f52626ffb
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Sep 14 11:03:43 2016 -0400
9986: Report actual thread shutdown wait if >0.1s (instead of just saying it took >1s)
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 5e0e4c7..20192f9 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -67,17 +67,16 @@ class MountTestBase(unittest.TestCase):
def tearDown(self):
if self.llfuse_thread:
subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
- self.llfuse_thread.join(timeout=1)
- if self.llfuse_thread.is_alive():
- logger.warning("MountTestBase.tearDown():"
- " llfuse thread still alive 1s after umount"
- " -- waiting another 10s")
- self.llfuse_thread.join(timeout=10)
+ t0 = time.time()
+ self.llfuse_thread.join(timeout=10)
if self.llfuse_thread.is_alive():
logger.warning("MountTestBase.tearDown():"
" llfuse thread still alive 10s after umount"
" -- exiting with SIGKILL")
os.kill(os.getpid(), signal.SIGKILL)
+ waited = time.time() - t0
+ if waited > 0.1:
+ logger.warning("MountTestBase.tearDown(): waited %f s for llfuse thread to end", waited)
os.rmdir(self.mounttmp)
if self.keeptmp:
commit 7382b7a2d35e93944ff364d36017f51e9a1cd392
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 8 14:26:58 2016 -0400
9986: Kill test suite when danger of deadlock is detected.
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index d0e3102..c705934 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -76,7 +76,12 @@ class IntegrationTest(unittest.TestCase):
arvados_fuse.command.ArgumentParser().parse_args(
argv + ['--foreground',
'--unmount-timeout=0.1',
- self.mnt])):
+ self.mnt])) as m:
return func(self, *args, **kwargs)
+ if m.llfuse_thread.is_alive():
+ self.logger.warning("IntegrationTest.mount:"
+ " llfuse thread still alive after umount"
+ " -- killing test suite to avoid deadlock")
+ os.kill(os.getpid(), signal.SIGKILL)
return wrapper
return decorator
commit caa4cce0bc97ae99bc1657c3126d3c098e959eaa
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 8 09:34:31 2016 -0400
9986: Share one multiprocessing pool with all IntegrationTests.
diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 5a45bfc..d0e3102 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -1,14 +1,28 @@
import arvados
import arvados_fuse
import arvados_fuse.command
+import atexit
import functools
import inspect
import multiprocessing
import os
+import run_test_server
+import signal
import sys
import tempfile
import unittest
-import run_test_server
+
+_pool = None
+
+
+ at atexit.register
+def _pool_cleanup():
+ global _pool
+ if _pool is None:
+ return
+ _pool.close()
+ _pool.join()
+
def wrap_static_test_method(modName, clsName, funcName, args, kwargs):
class Test(unittest.TestCase):
@@ -24,17 +38,15 @@ class IntegrationTest(unittest.TestCase):
If called by method 'foobar', the static method '_foobar' of
the same class will be called in the other process.
"""
+ global _pool
+ if _pool is None:
+ _pool = multiprocessing.Pool(1, maxtasksperchild=1)
modName = inspect.getmodule(self).__name__
clsName = self.__class__.__name__
funcName = inspect.currentframe().f_back.f_code.co_name
- pool = multiprocessing.Pool(1)
- try:
- pool.apply(
- wrap_static_test_method,
- (modName, clsName, '_'+funcName, args, kwargs))
- finally:
- pool.terminate()
- pool.join()
+ _pool.apply(
+ wrap_static_test_method,
+ (modName, clsName, '_'+funcName, args, kwargs))
@classmethod
def setUpClass(cls):
commit 89c7264f5a3b53043762fb8ced7bdb997ed2a120
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Sep 8 08:56:18 2016 -0400
9986: Send SIGKILL if llfuse thread does not join.
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 12395d7..5e0e4c7 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -1,18 +1,19 @@
import arvados
-import arvados.safeapi
import arvados_fuse as fuse
+import arvados.safeapi
import llfuse
+import logging
+import multiprocessing
import os
+import run_test_server
import shutil
+import signal
import subprocess
import sys
import tempfile
import threading
import time
import unittest
-import logging
-import multiprocessing
-import run_test_server
logger = logging.getLogger('arvados.arv-mount')
@@ -64,23 +65,27 @@ class MountTestBase(unittest.TestCase):
return self.operations.inodes[llfuse.ROOT_INODE]
def tearDown(self):
- self.pool.terminate()
- self.pool.join()
- del self.pool
-
if self.llfuse_thread:
subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
self.llfuse_thread.join(timeout=1)
if self.llfuse_thread.is_alive():
logger.warning("MountTestBase.tearDown():"
" llfuse thread still alive 1s after umount"
- " -- abandoning and exiting anyway")
+ " -- waiting another 10s")
+ self.llfuse_thread.join(timeout=10)
+ if self.llfuse_thread.is_alive():
+ logger.warning("MountTestBase.tearDown():"
+ " llfuse thread still alive 10s after umount"
+ " -- exiting with SIGKILL")
+ os.kill(os.getpid(), signal.SIGKILL)
os.rmdir(self.mounttmp)
if self.keeptmp:
shutil.rmtree(self.keeptmp)
os.environ.pop('KEEP_LOCAL_STORE')
run_test_server.reset()
+ self.pool.close()
+ self.pool.join()
def assertDirContents(self, subdir, expect_content):
path = self.mounttmp
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list