[arvados] updated: 2.7.0-5343-g7de94b0902
git repository hosting
git at public.arvados.org
Fri Nov 10 21:08:00 UTC 2023
Summary of changes:
sdk/python/tests/run_test_server.py | 55 +++++++++++++++++++++++++++++--------
1 file changed, 43 insertions(+), 12 deletions(-)
via 7de94b0902e234b42f37c3c919aebfb929724f95 (commit)
via 172bf8620c2edb43ab3b71b45494712cbdffbfc0 (commit)
from 4ca9a41d612e81a54b216c98f808230995ea69d0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 7de94b0902e234b42f37c3c919aebfb929724f95
Author: Tom Clegg <tom at curii.com>
Date: Fri Nov 10 16:06:41 2023 -0500
20690: Detach background processes from process group.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index a5dd88a9c5..45c9630266 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -256,17 +256,20 @@ def _logfilename(label):
cat = subprocess.Popen(
stdbuf+['cat', fifo],
stdin=open('/dev/null'),
- stdout=subprocess.PIPE)
+ stdout=subprocess.PIPE,
+ start_new_session=True)
_detachedSubprocesses.append(cat)
tee = subprocess.Popen(
stdbuf+['tee', '-a', logfilename],
stdin=cat.stdout,
- stdout=subprocess.PIPE)
+ stdout=subprocess.PIPE,
+ start_new_session=True)
_detachedSubprocesses.append(tee)
sed = subprocess.Popen(
stdbuf+['sed', '-e', 's/^/['+label+'] /'],
stdin=tee.stdout,
- stdout=sys.stderr)
+ stdout=sys.stderr,
+ start_new_session=True)
_detachedSubprocesses.append(sed)
return fifo
@@ -378,7 +381,8 @@ def run(leave_running_atexit=False):
'--ssl',
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
- env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf,
+ start_new_session=True)
_detachedSubprocesses.append(railsapi)
if not leave_running_atexit:
@@ -456,7 +460,8 @@ def run_controller():
port = internal_port_from_config("Controller")
controller = subprocess.Popen(
["arvados-server", "controller"],
- stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True,
+ start_new_session=True)
_detachedSubprocesses.append(controller)
with open(_pidfile('controller'), 'w') as f:
f.write(str(controller.pid))
@@ -476,7 +481,8 @@ def run_ws():
logf = open(_logfilename('ws'), WRITE_MODE)
ws = subprocess.Popen(
["arvados-server", "ws"],
- stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True,
+ start_new_session=True)
_detachedSubprocesses.append(ws)
with open(_pidfile('ws'), 'w') as f:
f.write(str(ws.pid))
@@ -510,7 +516,8 @@ def _start_keep(n, blob_signing=False):
with open(_logfilename('keep{}'.format(n)), WRITE_MODE) as logf:
with open('/dev/null') as _stdin:
child = subprocess.Popen(
- keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
+ keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True,
+ start_new_session=True)
_detachedSubprocesses.append(child)
print('child.pid is %d'%child.pid, file=sys.stderr)
@@ -577,7 +584,8 @@ def run_keep_proxy():
env['ARVADOS_API_TOKEN'] = auth_token('anonymous')
logf = open(_logfilename('keepproxy'), WRITE_MODE)
kp = subprocess.Popen(
- ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ ['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True,
+ start_new_session=True)
_detachedSubprocesses.append(kp)
with open(_pidfile('keepproxy'), 'w') as f:
@@ -617,7 +625,8 @@ def run_arv_git_httpd():
env.pop('ARVADOS_API_TOKEN', None)
logf = open(_logfilename('githttpd'), WRITE_MODE)
agh = subprocess.Popen(['arvados-server', 'git-httpd'],
- env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf,
+ start_new_session=True)
_detachedSubprocesses.append(agh)
with open(_pidfile('githttpd'), 'w') as f:
f.write(str(agh.pid))
@@ -638,7 +647,8 @@ def run_keep_web():
logf = open(_logfilename('keep-web'), WRITE_MODE)
keepweb = subprocess.Popen(
['arvados-server', 'keep-web'],
- env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf,
+ start_new_session=True)
_detachedSubprocesses.append(keepweb)
with open(_pidfile('keep-web'), 'w') as f:
f.write(str(keepweb.pid))
@@ -696,7 +706,8 @@ def run_nginx():
['nginx',
'-g', 'error_log stderr info; pid '+_pidfile('nginx')+';',
'-c', conffile],
- env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ env=env, stdin=open('/dev/null'), stdout=sys.stderr,
+ start_new_session=True)
_detachedSubprocesses.append(nginx)
_wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
commit 172bf8620c2edb43ab3b71b45494712cbdffbfc0
Author: Tom Clegg <tom at curii.com>
Date: Fri Nov 10 14:48:43 2023 -0500
20690: Silence useless ResourceWarning messages.
(Not especially related to issue 20690)
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index eb2784c714..a5dd88a9c5 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -41,6 +41,15 @@ if __name__ == '__main__' and os.path.exists(
import arvados
import arvados.config
+# This module starts subprocesses and records them in pidfiles so they
+# can be managed by other processes (incl. after this process
+# exits). But if we don't keep a reference to each subprocess object
+# somewhere, the subprocess destructor runs, and we get a lot of
+# ResourceWarning noise in test logs. This is our bucket of subprocess
+# objects whose destructors we don't want to run but are otherwise
+# unneeded.
+_detachedSubprocesses = []
+
ARVADOS_DIR = os.path.realpath(os.path.join(MY_DIRNAME, '../../..'))
SERVICES_SRC_DIR = os.path.join(ARVADOS_DIR, 'services')
@@ -248,14 +257,17 @@ def _logfilename(label):
stdbuf+['cat', fifo],
stdin=open('/dev/null'),
stdout=subprocess.PIPE)
+ _detachedSubprocesses.append(cat)
tee = subprocess.Popen(
stdbuf+['tee', '-a', logfilename],
stdin=cat.stdout,
stdout=subprocess.PIPE)
- subprocess.Popen(
+ _detachedSubprocesses.append(tee)
+ sed = subprocess.Popen(
stdbuf+['sed', '-e', 's/^/['+label+'] /'],
stdin=tee.stdout,
stdout=sys.stderr)
+ _detachedSubprocesses.append(sed)
return fifo
def run(leave_running_atexit=False):
@@ -367,6 +379,7 @@ def run(leave_running_atexit=False):
'--ssl-certificate', 'tmp/self-signed.pem',
'--ssl-certificate-key', 'tmp/self-signed.key'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ _detachedSubprocesses.append(railsapi)
if not leave_running_atexit:
atexit.register(kill_server_pid, pid_file, passenger_root=api_src_dir)
@@ -444,6 +457,7 @@ def run_controller():
controller = subprocess.Popen(
["arvados-server", "controller"],
stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(controller)
with open(_pidfile('controller'), 'w') as f:
f.write(str(controller.pid))
_wait_until_port_listens(port)
@@ -463,6 +477,7 @@ def run_ws():
ws = subprocess.Popen(
["arvados-server", "ws"],
stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(ws)
with open(_pidfile('ws'), 'w') as f:
f.write(str(ws.pid))
_wait_until_port_listens(port)
@@ -496,6 +511,7 @@ def _start_keep(n, blob_signing=False):
with open('/dev/null') as _stdin:
child = subprocess.Popen(
keep_cmd, stdin=_stdin, stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(child)
print('child.pid is %d'%child.pid, file=sys.stderr)
with open(_pidfile('keep{}'.format(n)), 'w') as f:
@@ -562,6 +578,7 @@ def run_keep_proxy():
logf = open(_logfilename('keepproxy'), WRITE_MODE)
kp = subprocess.Popen(
['arvados-server', 'keepproxy'], env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf, close_fds=True)
+ _detachedSubprocesses.append(kp)
with open(_pidfile('keepproxy'), 'w') as f:
f.write(str(kp.pid))
@@ -601,6 +618,7 @@ def run_arv_git_httpd():
logf = open(_logfilename('githttpd'), WRITE_MODE)
agh = subprocess.Popen(['arvados-server', 'git-httpd'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ _detachedSubprocesses.append(agh)
with open(_pidfile('githttpd'), 'w') as f:
f.write(str(agh.pid))
_wait_until_port_listens(gitport)
@@ -621,6 +639,7 @@ def run_keep_web():
keepweb = subprocess.Popen(
['arvados-server', 'keep-web'],
env=env, stdin=open('/dev/null'), stdout=logf, stderr=logf)
+ _detachedSubprocesses.append(keepweb)
with open(_pidfile('keep-web'), 'w') as f:
f.write(str(keepweb.pid))
_wait_until_port_listens(keepwebport)
@@ -678,6 +697,7 @@ def run_nginx():
'-g', 'error_log stderr info; pid '+_pidfile('nginx')+';',
'-c', conffile],
env=env, stdin=open('/dev/null'), stdout=sys.stderr)
+ _detachedSubprocesses.append(nginx)
_wait_until_port_listens(nginxconf['CONTROLLERSSLPORT'])
def setup_config():
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list