Skip to content

Commit b02aefc

Browse files
rani-pinchukgpshead
rani-pinchuk
authored and
gpshead
committed
gh-135427: Fix DeprecationWarning for os.fork when run in threads with -Werror (GH-136796)
Don't ignore errors raised by `PyErr_WarnFormat` in `warn_about_fork_with_threads` Instead, ignore the warnings in all test code that forks. (That's a lot of functions.) In `test_support`, make `ignore_warnings` a context manager (as well as decorator), and add a `message` argument to it. Also add a `ignore_fork_in_thread_deprecation_warnings` helper for the deadlock-in-fork warning.
1 parent 504bf76 commit b02aefc

27 files changed

Lines changed: 422 additions & 107 deletions

‎Lib/test/_test_multiprocessing.py‎

Lines changed: 204 additions & 11 deletions
Large diffs are not rendered by default.

‎Lib/test/support/warnings_helper.py‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import contextlib
2-
import functools
32
import importlib
43
import re
54
import sys
65
import warnings
76

87

8+
99
def import_deprecated(name):
1010
"""Import *name* while suppressing DeprecationWarning."""
1111
with warnings.catch_warnings():
@@ -42,20 +42,32 @@ def check_syntax_warning(testcase, statement, errtext='',
4242
testcase.assertEqual(warns, [])
4343

4444

45-
def ignore_warnings(*, category):
45+
@contextlib.contextmanager
46+
def ignore_warnings(*, category, message=''):
4647
"""Decorator to suppress warnings.
4748
48-
Use of context managers to hide warnings make diffs
49-
more noisy and tools like 'git blame' less useful.
49+
Can also be used as a context manager. This is not preferred,
50+
because it makes diffs more noisy and tools like 'git blame' less useful.
51+
But, it's useful for async functions.
5052
"""
51-
def decorator(test):
52-
@functools.wraps(test)
53-
def wrapper(self, *args, **kwargs):
54-
with warnings.catch_warnings():
55-
warnings.simplefilter('ignore', category=category)
56-
return test(self, *args, **kwargs)
57-
return wrapper
58-
return decorator
53+
with warnings.catch_warnings():
54+
warnings.filterwarnings('ignore', category=category, message=message)
55+
yield
56+
57+
58+
@contextlib.contextmanager
59+
def ignore_fork_in_thread_deprecation_warnings():
60+
"""Suppress deprecation warnings related to forking in multi-threaded code.
61+
62+
See gh-135427
63+
64+
Can be used as decorator (preferred) or context manager.
65+
"""
66+
with ignore_warnings(
67+
message=".*fork.*may lead to deadlocks in the child.*",
68+
category=DeprecationWarning,
69+
):
70+
yield
5971

6072

6173
class WarningsRecorder(object):

‎Lib/test/test_asyncio/test_unix_events.py‎

Lines changed: 59 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from unittest import mock
1616

1717
from test import support
18-
from test.support import os_helper
18+
from test.support import os_helper, warnings_helper
1919
from test.support import socket_helper
2020
from test.support import wait_process
2121
from test.support import hashlib_helper
@@ -1180,67 +1180,63 @@ async def runner():
11801180

11811181

11821182
@support.requires_fork()
1183-
class TestFork(unittest.TestCase):
1184-
1185-
def test_fork_not_share_current_task(self):
1186-
loop = object()
1187-
task = object()
1188-
asyncio._set_running_loop(loop)
1189-
self.addCleanup(asyncio._set_running_loop, None)
1190-
asyncio.tasks._enter_task(loop, task)
1191-
self.addCleanup(asyncio.tasks._leave_task, loop, task)
1192-
self.assertIs(asyncio.current_task(), task)
1193-
r, w = os.pipe()
1194-
self.addCleanup(os.close, r)
1195-
self.addCleanup(os.close, w)
1196-
pid = os.fork()
1197-
if pid == 0:
1198-
# child
1199-
try:
1200-
asyncio._set_running_loop(loop)
1201-
current_task = asyncio.current_task()
1202-
if current_task is None:
1203-
os.write(w, b'NO TASK')
1204-
else:
1205-
os.write(w, b'TASK:' + str(id(current_task)).encode())
1206-
except BaseException as e:
1207-
os.write(w, b'ERROR:' + ascii(e).encode())
1208-
finally:
1209-
asyncio._set_running_loop(None)
1210-
os._exit(0)
1211-
else:
1212-
# parent
1213-
result = os.read(r, 100)
1214-
self.assertEqual(result, b'NO TASK')
1215-
wait_process(pid, exitcode=0)
1216-
1217-
def test_fork_not_share_event_loop(self):
1218-
# The forked process should not share the event loop with the parent
1219-
loop = object()
1220-
asyncio._set_running_loop(loop)
1221-
self.assertIs(asyncio.get_running_loop(), loop)
1222-
self.addCleanup(asyncio._set_running_loop, None)
1223-
r, w = os.pipe()
1224-
self.addCleanup(os.close, r)
1225-
self.addCleanup(os.close, w)
1226-
pid = os.fork()
1227-
if pid == 0:
1228-
# child
1229-
try:
1230-
loop = asyncio.get_event_loop()
1231-
os.write(w, b'LOOP:' + str(id(loop)).encode())
1232-
except RuntimeError:
1233-
os.write(w, b'NO LOOP')
1234-
except BaseException as e:
1235-
os.write(w, b'ERROR:' + ascii(e).encode())
1236-
finally:
1237-
os._exit(0)
1238-
else:
1239-
# parent
1240-
result = os.read(r, 100)
1241-
self.assertEqual(result, b'NO LOOP')
1242-
wait_process(pid, exitcode=0)
1183+
class TestFork(unittest.IsolatedAsyncioTestCase):
12431184

1185+
async def test_fork_not_share_current_task(self):
1186+
with warnings_helper.ignore_fork_in_thread_deprecation_warnings():
1187+
loop = asyncio.get_running_loop()
1188+
task = asyncio.current_task()
1189+
self.assertIsNotNone(task)
1190+
r, w = os.pipe()
1191+
self.addCleanup(os.close, r)
1192+
self.addCleanup(os.close, w)
1193+
pid = os.fork()
1194+
if pid == 0:
1195+
# child
1196+
try:
1197+
asyncio._set_running_loop(loop)
1198+
current_task = asyncio.current_task()
1199+
if current_task is None:
1200+
os.write(w, b'NO TASK')
1201+
else:
1202+
os.write(w, b'TASK:' + str(id(current_task)).encode())
1203+
except BaseException as e:
1204+
os.write(w, b'ERROR:' + ascii(e).encode())
1205+
finally:
1206+
asyncio._set_running_loop(None)
1207+
os._exit(0)
1208+
else:
1209+
# parent
1210+
result = os.read(r, 100)
1211+
self.assertEqual(result, b'NO TASK')
1212+
wait_process(pid, exitcode=0)
1213+
1214+
async def test_fork_not_share_event_loop(self):
1215+
with warnings_helper.ignore_fork_in_thread_deprecation_warnings():
1216+
# The forked process should not share the event loop with the parent
1217+
loop = asyncio.get_running_loop()
1218+
r, w = os.pipe()
1219+
self.addCleanup(os.close, r)
1220+
self.addCleanup(os.close, w)
1221+
pid = os.fork()
1222+
if pid == 0:
1223+
# child
1224+
try:
1225+
loop = asyncio.get_event_loop()
1226+
os.write(w, b'LOOP:' + str(id(loop)).encode())
1227+
except RuntimeError:
1228+
os.write(w, b'NO LOOP')
1229+
except BaseException as e:
1230+
os.write(w, b'ERROR:' + ascii(e).encode())
1231+
finally:
1232+
os._exit(0)
1233+
else:
1234+
# parent
1235+
result = os.read(r, 100)
1236+
self.assertEqual(result, b'NO LOOP')
1237+
wait_process(pid, exitcode=0)
1238+
1239+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
12441240
@hashlib_helper.requires_hashdigest('md5')
12451241
@support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
12461242
def test_fork_signal_handling(self):
@@ -1288,6 +1284,7 @@ async def func():
12881284
self.assertFalse(parent_handled.is_set())
12891285
self.assertTrue(child_handled.is_set())
12901286

1287+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
12911288
@hashlib_helper.requires_hashdigest('md5')
12921289
@support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
12931290
def test_fork_asyncio_run(self):
@@ -1308,6 +1305,7 @@ async def child_main():
13081305

13091306
self.assertEqual(result.value, 42)
13101307

1308+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
13111309
@hashlib_helper.requires_hashdigest('md5')
13121310
@support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
13131311
def test_fork_asyncio_subprocess(self):

‎Lib/test/test_builtin.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
from test import support
3232
from test.support import cpython_only, swap_attr
3333
from test.support import async_yield, run_yielding_async_fn
34+
from test.support import warnings_helper
3435
from test.support.import_helper import import_module
3536
from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink)
3637
from test.support.script_helper import assert_python_ok
@@ -2545,6 +2546,7 @@ def run_child(self, child, terminal_input):
25452546
finally:
25462547
signal.signal(signal.SIGHUP, old_sighup)
25472548

2549+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
25482550
def _run_child(self, child, terminal_input):
25492551
r, w = os.pipe() # Pipe test results from child back to parent
25502552
try:

‎Lib/test/test_concurrent_futures/executor.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from concurrent import futures
66
from operator import add
77
from test import support
8-
from test.support import Py_GIL_DISABLED
8+
from test.support import Py_GIL_DISABLED, warnings_helper
99

1010

1111
def mul(x, y):
@@ -43,10 +43,12 @@ class ExecutorTest:
4343

4444
# Executor.shutdown() and context manager usage is tested by
4545
# ExecutorShutdownTest.
46+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
4647
def test_submit(self):
4748
future = self.executor.submit(pow, 2, 8)
4849
self.assertEqual(256, future.result())
4950

51+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
5052
def test_submit_keyword(self):
5153
future = self.executor.submit(mul, 2, y=8)
5254
self.assertEqual(16, future.result())
@@ -57,6 +59,7 @@ def test_submit_keyword(self):
5759
with self.assertRaises(TypeError):
5860
self.executor.submit(arg=1)
5961

62+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
6063
def test_map(self):
6164
self.assertEqual(
6265
list(self.executor.map(pow, range(10), range(10))),
@@ -66,13 +69,15 @@ def test_map(self):
6669
list(self.executor.map(pow, range(10), range(10), chunksize=3)),
6770
list(map(pow, range(10), range(10))))
6871

72+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
6973
def test_map_exception(self):
7074
i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5])
7175
self.assertEqual(i.__next__(), (0, 1))
7276
self.assertEqual(i.__next__(), (0, 1))
7377
with self.assertRaises(ZeroDivisionError):
7478
i.__next__()
7579

80+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
7681
@support.requires_resource('walltime')
7782
def test_map_timeout(self):
7883
results = []
@@ -108,26 +113,30 @@ def test_map_buffersize_value_validation(self):
108113
):
109114
self.executor.map(str, range(4), buffersize=buffersize)
110115

116+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
111117
def test_map_buffersize(self):
112118
ints = range(4)
113119
for buffersize in (1, 2, len(ints), len(ints) * 2):
114120
with self.subTest(buffersize=buffersize):
115121
res = self.executor.map(str, ints, buffersize=buffersize)
116122
self.assertListEqual(list(res), ["0", "1", "2", "3"])
117123

124+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
118125
def test_map_buffersize_on_multiple_iterables(self):
119126
ints = range(4)
120127
for buffersize in (1, 2, len(ints), len(ints) * 2):
121128
with self.subTest(buffersize=buffersize):
122129
res = self.executor.map(add, ints, ints, buffersize=buffersize)
123130
self.assertListEqual(list(res), [0, 2, 4, 6])
124131

132+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
125133
def test_map_buffersize_on_infinite_iterable(self):
126134
res = self.executor.map(str, itertools.count(), buffersize=2)
127135
self.assertEqual(next(res, None), "0")
128136
self.assertEqual(next(res, None), "1")
129137
self.assertEqual(next(res, None), "2")
130138

139+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
131140
def test_map_buffersize_on_multiple_infinite_iterables(self):
132141
res = self.executor.map(
133142
add,
@@ -147,6 +156,7 @@ def test_map_buffersize_without_iterable(self):
147156
res = self.executor.map(str, buffersize=2)
148157
self.assertIsNone(next(res, None))
149158

159+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
150160
def test_map_buffersize_when_buffer_is_full(self):
151161
ints = iter(range(4))
152162
buffersize = 2
@@ -158,13 +168,15 @@ def test_map_buffersize_when_buffer_is_full(self):
158168
msg="should have fetched only `buffersize` elements from `ints`.",
159169
)
160170

171+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
161172
def test_shutdown_race_issue12456(self):
162173
# Issue #12456: race condition at shutdown where trying to post a
163174
# sentinel in the call queue blocks (the queue is full while processes
164175
# have exited).
165176
self.executor.map(str, [2] * (self.worker_count + 1))
166177
self.executor.shutdown()
167178

179+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
168180
@support.cpython_only
169181
def test_no_stale_references(self):
170182
# Issue #16284: check that the executors don't unnecessarily hang onto
@@ -209,6 +221,7 @@ def test_max_workers_negative(self):
209221
"than 0"):
210222
self.executor_type(max_workers=number)
211223

224+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
212225
def test_free_reference(self):
213226
# Issue #14406: Result iterator should not keep an internal
214227
# reference to result objects.
@@ -221,6 +234,7 @@ def test_free_reference(self):
221234
if wr() is None:
222235
break
223236

237+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
224238
def test_swallows_falsey_exceptions(self):
225239
# see gh-132063: Prevent exceptions that evaluate as falsey
226240
# from being ignored.

‎Lib/test/test_concurrent_futures/test_as_completed.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
CANCELLED_AND_NOTIFIED, FINISHED, Future)
88

99
from test import support
10+
from test.support import warnings_helper
1011

1112
from .util import (
1213
PENDING_FUTURE, RUNNING_FUTURE,
@@ -19,6 +20,7 @@ def mul(x, y):
1920

2021

2122
class AsCompletedTests:
23+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
2224
def test_no_timeout(self):
2325
future1 = self.executor.submit(mul, 2, 21)
2426
future2 = self.executor.submit(mul, 7, 6)
@@ -35,6 +37,7 @@ def test_no_timeout(self):
3537
future1, future2]),
3638
completed)
3739

40+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
3841
def test_future_times_out(self):
3942
"""Test ``futures.as_completed`` timing out before
4043
completing it's final future."""
@@ -62,6 +65,7 @@ def test_future_times_out(self):
6265
# Check that ``future`` wasn't completed.
6366
self.assertEqual(completed_futures, already_completed)
6467

68+
@warnings_helper.ignore_fork_in_thread_deprecation_warnings()
6569
def test_duplicate_futures(self):
6670
# Issue 20367. Duplicate futures should not raise exceptions or give
6771
# duplicate responses.

0 commit comments

Comments
 (0)