[ARVADOS] created: 5ac2b460fdfc000398415dfa50a5ac3bfedce0d7

Git user git at public.curoverse.com
Wed Apr 20 10:03:20 EDT 2016


        at  5ac2b460fdfc000398415dfa50a5ac3bfedce0d7 (commit)


commit 5ac2b460fdfc000398415dfa50a5ac3bfedce0d7
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Apr 18 09:27:01 2016 -0400

    8345: Use multiprocessing pool to get dir listings.

diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 55d3092..e573ddf 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -47,6 +47,10 @@ class MountTestBase(unittest.TestCase):
             raise
         llfuse.close()
 
+    def listmountdir(self, *subdirs):
+        return self.pool.apply(os.listdir,
+                               (os.path.join(self.mounttmp, *subdirs), ))
+
     def make_mount(self, root_class, **root_kwargs):
         self.operations = fuse.Operations(
             os.getuid(), os.getgid(),
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index e534e32..73da16d 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -127,7 +127,7 @@ class FuseMagicTest(MountTestBase):
     def runTest(self):
         self.make_mount(fuse.MagicDirectory)
 
-        mount_ls = llfuse.listdir(self.mounttmp)
+        mount_ls = self.listmountdir()
         self.assertIn('README', mount_ls)
         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
                              arvados.util.uuid_pattern.match(fn)
@@ -136,11 +136,10 @@ class FuseMagicTest(MountTestBase):
         self.assertDirContents(self.testcollection, ['thing1.txt'])
         self.assertDirContents(os.path.join('by_id', self.testcollection),
                                ['thing1.txt'])
-        mount_ls = llfuse.listdir(self.mounttmp)
+        mount_ls = self.listmountdir()
         self.assertIn('README', mount_ls)
         self.assertIn(self.testcollection, mount_ls)
-        self.assertIn(self.testcollection,
-                      llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
+        self.assertIn(self.testcollection, self.listmountdir('by_id'))
 
         files = {}
         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'
@@ -154,15 +153,15 @@ class FuseTagsTest(MountTestBase):
     def runTest(self):
         self.make_mount(fuse.TagsDirectory)
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         d1.sort()
         self.assertEqual(['foo_tag'], d1)
 
-        d2 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag'))
+        d2 = self.listmountdir('foo_tag')
         d2.sort()
         self.assertEqual(['zzzzz-4zz18-fy296fx3hot09f7'], d2)
 
-        d3 = llfuse.listdir(os.path.join(self.mounttmp, 'foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7'))
+        d3 = self.listmountdir('foo_tag', 'zzzzz-4zz18-fy296fx3hot09f7')
         d3.sort()
         self.assertEqual(['foo'], d3)
 
@@ -178,12 +177,12 @@ class FuseTagsUpdateTest(MountTestBase):
     def runTest(self):
         self.make_mount(fuse.TagsDirectory, poll_time=1)
 
-        self.assertIn('foo_tag', llfuse.listdir(self.mounttmp))
+        self.assertIn('foo_tag', self.listmountdir())
 
         bar_uuid = run_test_server.fixture('collections')['bar_file']['uuid']
         self.tag_collection(bar_uuid, 'fuse_test_tag')
         time.sleep(1)
-        self.assertIn('fuse_test_tag', llfuse.listdir(self.mounttmp))
+        self.assertIn('fuse_test_tag', self.listmountdir())
         self.assertDirContents('fuse_test_tag', [bar_uuid])
 
         baz_uuid = run_test_server.fixture('collections')['baz_file']['uuid']
@@ -204,14 +203,14 @@ class FuseSharedTest(MountTestBase):
         # shared_dirs is a list of the directories exposed
         # by fuse.SharedDirectory (i.e. any object visible
         # to the current user)
-        shared_dirs = llfuse.listdir(self.mounttmp)
+        shared_dirs = self.listmountdir()
         shared_dirs.sort()
         self.assertIn('FUSE User', shared_dirs)
 
         # fuse_user_objs is a list of the objects owned by the FUSE
         # test user (which present as files in the 'FUSE User'
         # directory)
-        fuse_user_objs = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User'))
+        fuse_user_objs = self.listmountdir('FUSE User')
         fuse_user_objs.sort()
         self.assertEqual(['FUSE Test Project',                    # project owned by user
                           'collection #1 owned by FUSE',          # collection owned by user
@@ -220,7 +219,7 @@ class FuseSharedTest(MountTestBase):
                       ], fuse_user_objs)
 
         # test_proj_files is a list of the files in the FUSE Test Project.
-        test_proj_files = llfuse.listdir(os.path.join(self.mounttmp, 'FUSE User', 'FUSE Test Project'))
+        test_proj_files = self.listmountdir('FUSE User', 'FUSE Test Project')
         test_proj_files.sort()
         self.assertEqual(['collection in FUSE project',
                           'pipeline instance in FUSE project.pipelineInstance',
@@ -255,10 +254,10 @@ class FuseHomeTest(MountTestBase):
         self.make_mount(fuse.ProjectDirectory,
                         project_object=self.api.users().current().execute())
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertIn('Unrestricted public data', d1)
 
-        d2 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data'))
+        d2 = self.listmountdir('Unrestricted public data')
         public_project = run_test_server.fixture('groups')[
             'anonymously_accessible_project']
         found_in = 0
@@ -278,14 +277,14 @@ class FuseHomeTest(MountTestBase):
         self.assertNotEqual(0, found_in)
         self.assertNotEqual(0, found_not_in)
 
-        d3 = llfuse.listdir(os.path.join(self.mounttmp, 'Unrestricted public data', 'GNU General Public License, version 3'))
+        d3 = self.listmountdir('Unrestricted public data', 'GNU General Public License, version 3')
         self.assertEqual(["GNU_General_Public_License,_version_3.pdf"], d3)
 
 
 def fuseModifyFileTestHelperReadStartContents(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
-            d1 = llfuse.listdir(mounttmp)
+            d1 = os.listdir(mounttmp)
             self.assertEqual(["file1.txt"], d1)
             with open(os.path.join(mounttmp, "file1.txt")) as f:
                 self.assertEqual("blub", f.read())
@@ -294,7 +293,7 @@ def fuseModifyFileTestHelperReadStartContents(mounttmp):
 def fuseModifyFileTestHelperReadEndContents(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
-            d1 = llfuse.listdir(mounttmp)
+            d1 = os.listdir(mounttmp)
             self.assertEqual(["file1.txt"], d1)
             with open(os.path.join(mounttmp, "file1.txt")) as f:
                 self.assertEqual("plnp", f.read())
@@ -332,13 +331,13 @@ class FuseAddFileToCollectionTest(MountTestBase):
         with llfuse.lock:
             m.new_collection(collection.api_response(), collection)
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertEqual(["file1.txt"], d1)
 
         with collection.open("file2.txt", "w") as f:
             f.write("plnp")
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 
 
@@ -357,12 +356,12 @@ class FuseRemoveFileFromCollectionTest(MountTestBase):
         with llfuse.lock:
             m.new_collection(collection.api_response(), collection)
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertEqual(["file1.txt", "file2.txt"], sorted(d1))
 
         collection.remove("file2.txt")
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertEqual(["file1.txt"], d1)
 
 
@@ -394,7 +393,7 @@ class FuseCreateFileTest(MountTestBase):
 
         self.assertIn("file1.txt", collection)
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertEqual(["file1.txt"], d1)
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -494,13 +493,13 @@ def fuseMkdirTestHelper(mounttmp):
             with self.assertRaises(OSError):
                 os.mkdir(os.path.join(mounttmp, "testdir"))
 
-            d1 = llfuse.listdir(mounttmp)
+            d1 = os.listdir(mounttmp)
             self.assertEqual(["testdir"], d1)
 
             with open(os.path.join(mounttmp, "testdir", "file1.txt"), "w") as f:
                 f.write("Hello world!")
 
-            d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+            d1 = os.listdir(os.path.join(mounttmp, "testdir"))
             self.assertEqual(["file1.txt"], d1)
 
     Test().runTest()
@@ -539,14 +538,14 @@ def fuseRmTestHelperDeleteFile(mounttmp):
             with self.assertRaises(OSError):
                 os.rmdir(os.path.join(mounttmp, "testdir"))
 
-            d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+            d1 = os.listdir(os.path.join(mounttmp, "testdir"))
             self.assertEqual(["file1.txt"], d1)
 
             # Delete file
             os.remove(os.path.join(mounttmp, "testdir", "file1.txt"))
 
             # Make sure it's empty
-            d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+            d1 = os.listdir(os.path.join(mounttmp, "testdir"))
             self.assertEqual([], d1)
 
             # Try to delete it again
@@ -562,7 +561,7 @@ def fuseRmTestHelperRmdir(mounttmp):
             os.rmdir(os.path.join(mounttmp, "testdir"))
 
             # Make sure it's empty
-            d1 = llfuse.listdir(os.path.join(mounttmp))
+            d1 = os.listdir(os.path.join(mounttmp))
             self.assertEqual([], d1)
 
             # Try to delete it again
@@ -613,16 +612,16 @@ def fuseMvFileTestHelperWriteFile(mounttmp):
 def fuseMvFileTestHelperMoveFile(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
-            d1 = llfuse.listdir(os.path.join(mounttmp))
+            d1 = os.listdir(os.path.join(mounttmp))
             self.assertEqual(["testdir"], d1)
-            d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+            d1 = os.listdir(os.path.join(mounttmp, "testdir"))
             self.assertEqual(["file1.txt"], d1)
 
             os.rename(os.path.join(mounttmp, "testdir", "file1.txt"), os.path.join(mounttmp, "file1.txt"))
 
-            d1 = llfuse.listdir(os.path.join(mounttmp))
+            d1 = os.listdir(os.path.join(mounttmp))
             self.assertEqual(["file1.txt", "testdir"], sorted(d1))
-            d1 = llfuse.listdir(os.path.join(mounttmp, "testdir"))
+            d1 = os.listdir(os.path.join(mounttmp, "testdir"))
             self.assertEqual([], d1)
 
     Test().runTest()
@@ -678,16 +677,16 @@ class FuseRenameTest(MountTestBase):
         self.assertRegexpMatches(collection2["manifest_text"],
             r'\./testdir 86fb269d190d2c85f6e0468ceca42a20\+12\+A\S+ 0:12:file1\.txt$')
 
-        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        d1 = self.listmountdir()
         self.assertEqual(["testdir"], d1)
-        d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir"))
+        d1 = self.listmountdir("testdir")
         self.assertEqual(["file1.txt"], d1)
 
         os.rename(os.path.join(self.mounttmp, "testdir"), os.path.join(self.mounttmp, "testdir2"))
 
-        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        d1 = self.listmountdir()
         self.assertEqual(["testdir2"], sorted(d1))
-        d1 = llfuse.listdir(os.path.join(self.mounttmp, "testdir2"))
+        d1 = self.listmountdir("testdir2")
         self.assertEqual(["file1.txt"], d1)
 
         collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
@@ -706,7 +705,7 @@ class FuseUpdateFromEventTest(MountTestBase):
 
         self.operations.listen_for_events()
 
-        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        d1 = self.listmountdir()
         self.assertEqual([], sorted(d1))
 
         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
@@ -717,7 +716,7 @@ class FuseUpdateFromEventTest(MountTestBase):
 
         # should show up via event bus notify
 
-        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        d1 = self.listmountdir()
         self.assertEqual(["file1.txt"], sorted(d1))
 
 
@@ -727,7 +726,7 @@ def fuseFileConflictTestHelper(mounttmp):
             with open(os.path.join(mounttmp, "file1.txt"), "w") as f:
                 f.write("bar")
 
-            d1 = sorted(llfuse.listdir(os.path.join(mounttmp)))
+            d1 = sorted(os.listdir(os.path.join(mounttmp)))
             self.assertEqual(len(d1), 2)
 
             with open(os.path.join(mounttmp, "file1.txt"), "r") as f:
@@ -750,7 +749,7 @@ class FuseFileConflictTest(MountTestBase):
         with llfuse.lock:
             m.new_collection(collection.api_response(), collection)
 
-        d1 = llfuse.listdir(os.path.join(self.mounttmp))
+        d1 = self.listmountdir()
         self.assertEqual([], sorted(d1))
 
         with arvados.collection.Collection(collection.manifest_locator(), api_client=self.api) as collection2:
@@ -767,12 +766,12 @@ def fuseUnlinkOpenFileTest(mounttmp):
             with open(os.path.join(mounttmp, "file1.txt"), "w+") as f:
                 f.write("foo")
 
-                d1 = llfuse.listdir(os.path.join(mounttmp))
+                d1 = os.listdir(os.path.join(mounttmp))
                 self.assertEqual(["file1.txt"], sorted(d1))
 
                 os.remove(os.path.join(mounttmp, "file1.txt"))
 
-                d1 = llfuse.listdir(os.path.join(mounttmp))
+                d1 = os.listdir(os.path.join(mounttmp))
                 self.assertEqual([], sorted(d1))
 
                 f.seek(0)
@@ -955,29 +954,29 @@ class FuseProjectMkdirRmdirTest(MountTestBase):
         self.make_mount(fuse.ProjectDirectory,
                         project_object=self.api.users().current().execute())
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertNotIn('testcollection', d1)
 
         self.pool.apply(fuseProjectMkdirTestHelper1, (self.mounttmp,))
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertIn('testcollection', d1)
 
         self.pool.apply(fuseProjectMkdirTestHelper2, (self.mounttmp,))
 
-        d1 = llfuse.listdir(self.mounttmp)
+        d1 = self.listmountdir()
         self.assertNotIn('testcollection', d1)
 
 
 def fuseProjectMvTestHelper1(mounttmp):
     class Test(unittest.TestCase):
         def runTest(self):
-            d1 = llfuse.listdir(mounttmp)
+            d1 = os.listdir(mounttmp)
             self.assertNotIn('testcollection', d1)
 
             os.mkdir(os.path.join(mounttmp, "testcollection"))
 
-            d1 = llfuse.listdir(mounttmp)
+            d1 = os.listdir(mounttmp)
             self.assertIn('testcollection', d1)
 
             with self.assertRaises(OSError):
@@ -985,10 +984,10 @@ def fuseProjectMvTestHelper1(mounttmp):
 
             os.rename(os.path.join(mounttmp, "testcollection"), os.path.join(mounttmp, 'Unrestricted public data', 'testcollection'))
 
-            d1 = llfuse.listdir(mounttmp)
+            d1 = os.listdir(mounttmp)
             self.assertNotIn('testcollection', d1)
 
-            d1 = llfuse.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
+            d1 = os.listdir(os.path.join(mounttmp, 'Unrestricted public data'))
             self.assertIn('testcollection', d1)
 
     Test().runTest()
@@ -1030,9 +1029,9 @@ class MagicDirApiError(FuseMagicTest):
         self.operations.inodes.inode_cache.min_entries = 2
 
         with self.assertRaises(OSError):
-            llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
+            self.listmountdir(self.testcollection)
 
-        llfuse.listdir(os.path.join(self.mounttmp, self.testcollection))
+        self.listmountdir(self.testcollection)
 
 
 class FuseUnitTest(unittest.TestCase):
@@ -1087,7 +1086,7 @@ class FuseMagicTestPDHOnly(MountTestBase):
         else:
             self.make_mount(fuse.MagicDirectory, pdh_only=pdh_only)
 
-        mount_ls = llfuse.listdir(self.mounttmp)
+        mount_ls = self.listmountdir()
         self.assertIn('README', mount_ls)
         self.assertFalse(any(arvados.util.keep_locator_pattern.match(fn) or
                              arvados.util.uuid_pattern.match(fn)
@@ -1098,11 +1097,10 @@ class FuseMagicTestPDHOnly(MountTestBase):
         self.assertDirContents(self.testcollection, ['thing1.txt'])
         self.assertDirContents(os.path.join('by_id', self.testcollection),
                                ['thing1.txt'])
-        mount_ls = llfuse.listdir(self.mounttmp)
+        mount_ls = self.listmountdir()
         self.assertIn('README', mount_ls)
         self.assertIn(self.testcollection, mount_ls)
-        self.assertIn(self.testcollection,
-                      llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
+        self.assertIn(self.testcollection, self.listmountdir('by_id'))
 
         files = {}
         files[os.path.join(self.mounttmp, self.testcollection, 'thing1.txt')] = 'data 1'

commit 056f0547ab09021117e2c4200a613cae0ff1dd9c
Author: Tom Clegg <tom at curoverse.com>
Date:   Mon Apr 18 09:24:50 2016 -0400

    8345: Avoid starting a multiprocessing pool after starting threads.

diff --git a/services/fuse/tests/integration_test.py b/services/fuse/tests/integration_test.py
index 5a45bfc..b355093 100644
--- a/services/fuse/tests/integration_test.py
+++ b/services/fuse/tests/integration_test.py
@@ -9,6 +9,7 @@ import sys
 import tempfile
 import unittest
 import run_test_server
+from . import pool
 
 def wrap_static_test_method(modName, clsName, funcName, args, kwargs):
     class Test(unittest.TestCase):
@@ -27,14 +28,9 @@ class IntegrationTest(unittest.TestCase):
         modName = inspect.getmodule(self).__name__
         clsName = self.__class__.__name__
         funcName = inspect.currentframe().f_back.f_code.co_name
-        pool = multiprocessing.Pool(1)
-        try:
-            pool.apply(
-                wrap_static_test_method,
-                (modName, clsName, '_'+funcName, args, kwargs))
-        finally:
-            pool.terminate()
-            pool.join()
+        self.pool.apply(
+            wrap_static_test_method,
+            (modName, clsName, '_'+funcName, args, kwargs))
 
     @classmethod
     def setUpClass(cls):
@@ -46,6 +42,7 @@ class IntegrationTest(unittest.TestCase):
         run_test_server.stop_keep(num_servers=2)
 
     def setUp(self):
+        self.pool = pool.Pool()
         self.mnt = tempfile.mkdtemp()
         run_test_server.authorize_with('active')
         self.api = arvados.safeapi.ThreadSafeApiCache(arvados.config.settings())
@@ -63,7 +60,7 @@ class IntegrationTest(unittest.TestCase):
                 with arvados_fuse.command.Mount(
                         arvados_fuse.command.ArgumentParser().parse_args(
                             argv + ['--foreground',
-                                    '--unmount-timeout=0.1',
+                                    '--unmount-timeout=0.5',
                                     self.mnt])):
                     return func(self, *args, **kwargs)
             return wrapper
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 12395d7..55d3092 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -1,18 +1,17 @@
 import arvados
-import arvados.safeapi
 import arvados_fuse as fuse
+import arvados.safeapi
 import llfuse
+import logging
+import multiprocessing
 import os
+import run_test_server
 import shutil
 import subprocess
-import sys
 import tempfile
 import threading
-import time
 import unittest
-import logging
-import multiprocessing
-import run_test_server
+from . import pool
 
 logger = logging.getLogger('arvados.arv-mount')
 
@@ -25,7 +24,7 @@ class MountTestBase(unittest.TestCase):
         # deadlocks.  The workaround is to run some of our test code in a
         # separate process.  Forturnately the multiprocessing module makes this
         # relatively easy.
-        self.pool = multiprocessing.Pool(1)
+        self.pool = pool.Pool()
 
         if local_store:
             self.keeptmp = tempfile.mkdtemp()
@@ -64,10 +63,6 @@ class MountTestBase(unittest.TestCase):
         return self.operations.inodes[llfuse.ROOT_INODE]
 
     def tearDown(self):
-        self.pool.terminate()
-        self.pool.join()
-        del self.pool
-
         if self.llfuse_thread:
             subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
             self.llfuse_thread.join(timeout=1)
diff --git a/services/fuse/tests/pool.py b/services/fuse/tests/pool.py
new file mode 100644
index 0000000..0c655c3
--- /dev/null
+++ b/services/fuse/tests/pool.py
@@ -0,0 +1,64 @@
+import logging
+import multiprocessing
+import multiprocessing.pool
+import os
+import sys
+import threading
+try:
+    from . import stacktracer
+except:
+    pass
+
+"""Creating new pool workers (e.g., to run a test) after starting
+threads elsewhere in the program (e.g., in llfuse) might cause
+deadlock.
+
+(fork() can be called while some mutexes are locked in other threads:
+in the child process, those mutexes will never be released.)
+
+To avoid this, we create a static pool of multiprocessing workers up
+front, before we create any threads in our main process.  Also, if any
+of our pool's workers exit, we don't replace them with new workers.
+
+POOL_SIZE is our guess about how many workers will be enough.  This is
+1 + the number of workers that die during the test suite: i.e., if all
+tests run normally, POOL_SIZE=1 is enough: we use pool.apply(), which
+runs one task at a time.  If we guess too low, the test suite will
+fail with a suggestion to increase POOL_SIZE.
+"""
+POOL_SIZE = 1
+
+logger = logging.getLogger('arvados.arv-mount')
+_pool = None
+
+
+class _Pool(multiprocessing.pool.Pool):
+    """A Pool that doesn't replenish its worker pool."""
+
+    def _maintain_pool(self):
+        self._join_exited_workers()
+
+    def apply_async(self, *args, **kwargs):
+        if not self._pool:
+            sys.stderr.write("\n\nmultiprocessing pool is empty! "
+                             "increase POOL_SIZE.\n\n")
+            os._exit(1)
+        return super(_Pool, self).apply_async(*args, **kwargs)
+
+
+def Pool():
+    global _pool
+    if not _pool:
+        nthreads = threading.active_count()
+        if nthreads > 1:
+            logger = logging.getLogger('arvados.arv-mount')
+            logger.error("threading.active_count() is {} when "
+                         "creating multiprocessing.Pool.  Danger ahead!"
+                         "".format(nthreads))
+        _pool = _Pool(POOL_SIZE)
+        try:
+            stacktracer.trace_start("/tmp/trace.html", interval=1, auto=True)
+            pass
+        except:
+            pass
+    return _pool
diff --git a/services/fuse/tests/test_exec.py b/services/fuse/tests/test_exec.py
index 66013a4..f1f2fa7 100644
--- a/services/fuse/tests/test_exec.py
+++ b/services/fuse/tests/test_exec.py
@@ -5,6 +5,7 @@ import os
 import run_test_server
 import tempfile
 import unittest
+from . import pool
 
 try:
     from shlex import quote
@@ -40,16 +41,13 @@ class ExecMode(unittest.TestCase):
     def setUp(self):
         self.mnt = tempfile.mkdtemp()
         _, self.okfile = tempfile.mkstemp()
-        self.pool = multiprocessing.Pool(1)
 
     def tearDown(self):
-        self.pool.terminate()
-        self.pool.join()
         os.rmdir(self.mnt)
         os.unlink(self.okfile)
 
     def test_exec(self):
-        self.pool.apply(try_exec, (self.mnt, [
+        pool.Pool().apply(try_exec, (self.mnt, [
             'sh', '-c',
             'echo -n foo >{}; cp {} {}'.format(
                 quote(os.path.join(self.mnt, 'zzz', 'foo.txt')),

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list