[arvados] updated: 2.7.0-5344-g50bccff931
git repository hosting
git at public.arvados.org
Sat Nov 11 00:43:25 UTC 2023
Summary of changes:
sdk/python/tests/run_test_server.py | 22 +++++++++++++++++++++-
services/fuse/arvados_fuse/unmount.py | 10 ++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
via 50bccff93109fe6ef9c370d2a858049d8b2e22cd (commit)
via c68e4ba51336a871dff26ae9f8dc7eb7e316083d (commit)
via 3236e0cbbe0fcb9e684d63a0ffac4d1f4a0d96b8 (commit)
via ae4cf50d72adebc6df338ef54ec5f2755ad73b8f (commit)
from 10b08e0c05d176775c4f0cf13f6e77025bb1e636 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 50bccff93109fe6ef9c370d2a858049d8b2e22cd
Merge: ae4cf50d72 c68e4ba513
Author: Tom Clegg <tom at curii.com>
Date: Fri Nov 10 19:42:42 2023 -0500
Merge branch '20846-python-fixes'
refs #20846
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
commit c68e4ba51336a871dff26ae9f8dc7eb7e316083d
Author: Tom Clegg <tom at curii.com>
Date: Fri Nov 10 14:48:43 2023 -0500
20846: Silence ResourceWarning messages.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index eb2784c714..a5dd88a9c5 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -41,6 +41,15 @@ if __name__ == '__main__' and os.path.exists(
import arvados
import arvados.config
+# This module starts subprocesses and records them in pidfiles so they
+# can be managed by other processes (incl. after this process
+# exits). But if we don't keep a reference to each subprocess object
+# somewhere, the subprocess destructor runs, and we get a lot of
+# ResourceWarning noise in test logs. This is our bucket of subprocess
+# objects whose destructors we don't want to run but are otherwise
+# unneeded.
+_detachedSubprocesses = []
+
ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
@@ -248,14 +257,17 @@ def _logfilename(label):
stdbuf+['cat', fifo],
stdin=open('/dev/null'),
stdout=subprocess.PIPE)
+ _detachedSubprocesses.append(cat)
tee = subprocess.Popen(
stdbuf+['tee', '-a', logfilename],
stdin=cat.stdout,
stdout=subprocess.PIPE)
- subprocess.Popen(
+ _detachedSubprocesses.append(tee)
+ sed = subprocess.Popen(
stdbuf+['sed', '-e', 's/^/['+label+'] /'],
stdin=tee.stdout,
stdout=sys.stderr)
+ _detachedSubprocesses.append(sed)
return fifo
def run(leave_running_atexit=False):
@@ -367,6 +379,7 @@ def run(leave_running_atexit=False):
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ _detachedSubprocesses.append(railsapi)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
@@ -444,6 +457,7 @@ def run_controller():
controller = subprocess.Popen(
["arvados-server", "controller"],
stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(controller)
with open(_pidfile('controller'), 'w') as f:
f.write(str(controller.pid))
_wait_until_port_listens(port)
@@ -463,6 +477,7 @@ def run_ws():
ws = subprocess.Popen(
["arvados-server", "ws"],
stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(ws)
with open(_pidfile('ws'), 'w') as f:
f.write(str(ws.pid))
_wait_until_port_listens(port)
@@ -496,6 +511,7 @@ def _start_keep(n, blob_signing=False):
with open('/dev/null') as _stdin:
child = subprocess.Popen(
keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(child)
print('child.pid is %d'%child.pid, file=sys.stderr)
with open(_pidfile('keep{}'.format(n)), 'w') as f:
@@ -562,6 +578,7 @@ def run_keep_proxy():
logf = open(_logfilename('keepproxy'), WRITE_MODE)
kp = subprocess.Popen(
['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(kp)
with open(_pidfile('keepproxy'), 'w') as f:
f.write(str(kp.pid))
@@ -601,6 +618,7 @@ def run_arv_git_httpd():
logf = open(_logfilename('githttpd'), WRITE_MODE)
agh = subprocess.Popen(['arvados-server', 'git-httpd'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ _detachedSubprocesses.append(agh)
with open(_pidfile('githttpd'), 'w') as f:
f.write(str(agh.pid))
_wait_until_port_listens(gitport)
@@ -621,6 +639,7 @@ def run_keep_web():
keepweb = subprocess.Popen(
['arvados-server', 'keep-web'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ _detachedSubprocesses.append(keepweb)
with open(_pidfile('keep-web'), 'w') as f:
f.write(str(keepweb.pid))
_wait_until_port_listens(keepwebport)
@@ -678,6 +697,7 @@ def run_nginx():
'-g', 'error_log stderr info; pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _detachedSubprocesses.append(nginx)
_wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
def setup_config():
commit 3236e0cbbe0fcb9e684d63a0ffac4d1f4a0d96b8
Author: Tom Clegg <tom at curii.com>
Date: Fri Nov 10 19:39:28 2023 -0500
20846: Avoid deadlock in unmount-and-replace race.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/fuse/arvados_fuse/unmount.py b/services/fuse/arvados_fuse/unmount.py
index 12d047a8f3..144c582ddc 100644
--- a/services/fuse/arvados_fuse/unmount.py
+++ b/services/fuse/arvados_fuse/unmount.py
@@ -154,6 +154,16 @@ def unmount(path, subtype=None, timeout=10, recursive=False):
path = os.path.realpath(path)
continue
elif not mounted:
+ if was_mounted:
+ # This appears to avoid a race condition where we
+ # return control to the caller after running
+ # "fusermount -u -z" (see below), the caller (e.g.,
+ # arv-mount --replace) immediately tries to attach a
+ # new fuse mount at the same mount point, the
+ # lazy-unmount process unmounts that _new_ mount while
+ # it is being initialized, and the setup code waits
+ # forever for the new mount to be initialized.
+ time.sleep(1)
return was_mounted
if attempted:
commit ae4cf50d72adebc6df338ef54ec5f2755ad73b8f
Author: Tom Clegg <tom at curii.com>
Date: Fri Nov 10 17:18:13 2023 -0500
20846: Avoid deadlock in unmount-and-replace race.
refs #20846
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/fuse/arvados_fuse/unmount.py b/services/fuse/arvados_fuse/unmount.py
index 12d047a8f3..144c582ddc 100644
--- a/services/fuse/arvados_fuse/unmount.py
+++ b/services/fuse/arvados_fuse/unmount.py
@@ -154,6 +154,16 @@ def unmount(path, subtype=None, timeout=10, recursive=False):
path = os.path.realpath(path)
continue
elif not mounted:
+ if was_mounted:
+ # This appears to avoid a race condition where we
+ # return control to the caller after running
+ # "fusermount -u -z" (see below), the caller (e.g.,
+ # arv-mount --replace) immediately tries to attach a
+ # new fuse mount at the same mount point, the
+ # lazy-unmount process unmounts that _new_ mount while
+ # it is being initialized, and the setup code waits
+ # forever for the new mount to be initialized.
+ time.sleep(1)
return was_mounted
if attempted:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list