Skip to content

Commit c308344

Browse files
committed
pythongh-126434: Process signals on dedicated thread
1 parent c15a45a commit c308344

2 files changed

Lines changed: 89 additions & 62 deletions

File tree

‎Lib/multiprocessing/synchronize.py‎

Lines changed: 25 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
__all__ = [
1111
'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Condition', 'Event'
12-
]
12+
]
1313

1414
import threading
1515
import sys
@@ -157,7 +157,7 @@ def __repr__(self):
157157
except Exception:
158158
value = 'unknown'
159159
return '<%s(value=%s, maxvalue=%s)>' % \
160-
(self.__class__.__name__, value, self._semlock.maxvalue)
160+
(self.__class__.__name__, value, self._semlock.maxvalue)
161161

162162
#
163163
# Non-recursive lock
@@ -253,7 +253,7 @@ def __repr__(self):
253253

254254
def wait(self, timeout=None):
255255
assert self._lock._semlock._is_mine(), \
256-
'must acquire() condition before using wait()'
256+
'must acquire() condition before using wait()'
257257

258258
# indicate that this thread is going to sleep
259259
self._sleeping_count.release()
@@ -328,72 +328,41 @@ def wait_for(self, predicate, timeout=None):
328328
class Event(object):
329329

330330
def __init__(self, *, ctx):
331-
self._flag = ctx.Value('i', 0)
332-
# Allocate a ctypes.c_ulonglong to hold the set_id:
333-
# Represents the C unsigned long long datatype.
334-
# The constructor accepts an optional integer initializer; no overflow checking is done.
335-
# From https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong
336-
# See multiprocessing/sharedctypes.py for typecode to ctypes definitions
337-
self._set_id = ctx.Value('Q', 0)
331+
self._cond = ctx.Condition(ctx.Lock())
332+
self._flag = ctx.Semaphore(0)
338333

339334
def is_set(self):
340-
# From https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Value :
341-
# If lock is True (the default) then a new recursive lock object is created to synchronize access to the value.
342-
with self._flag:
343-
return self._flag.value == 1
335+
with self._cond:
336+
if self._flag.acquire(False):
337+
self._flag.release()
338+
return True
339+
return False
344340

345341
def set(self):
346-
# If set() is called from a signal handler, this is fine as the lock is recursive (i.e. it won't deadlock).
347-
# If the thread interrupted by the signal handler is wait()-ing and the signal handler calls set(),
348-
# this is fine as wait() spins on the value. Thus, after the signal handler is done, the thread will
349-
# return from wait()
350-
# Fixes https://github.com/python/cpython/issues/85772
351-
with self._flag:
352-
with self._set_id:
353-
if self._flag.value == 0:
354-
# There is a theoretical chance of a race here. It requires the following conditions:
355-
# The interrupted thread must be wait()ing.
356-
# Then set() must be called reentrant for the maximum value of c_ulonglong + 1 times, i.e.
357-
# ./python.exe -c 'from ctypes import *; print(256**sizeof(c_ulonglong()))', which
358-
# evaluates to 18446744073709551616 on a 64 bit machine.
359-
# All interruptions must happen exactly after `if self._flag.value == 0:`.
360-
# The _set_id value will then wrap around. Then clear() must be called
361-
# before the original wait() code continues. The wait() code will then continue
362-
# to (incorrectly) wait. This case should be safe to ignore. The stack
363-
# will probably grow too large before there is any chance of this actually happening.
364-
365-
self._flag.value = 1
366-
self._set_id.value += 1
367-
# There is no race here by reentrant set() when reaching the maximum value for `self._set_id.value`.
368-
# ctypes.c_ulonglong will overflow without any exception:
369-
# https://docs.python.org/3/library/ctypes.html#ctypes.c_ulonglong
370-
# > no overflow checking is done.
371-
# This means that we do not need to check if some maximum value is reached:
372-
# C will wrap around the value for us.
342+
with self._cond:
343+
self._flag.acquire(False)
344+
self._flag.release()
345+
self._cond.notify_all()
373346

374347
def clear(self):
375-
with self._flag:
376-
self._flag.value = 0
348+
with self._cond:
349+
self._flag.acquire(False)
377350

378351
def wait(self, timeout=None):
379-
start_time = time.monotonic()
380-
set_id = self._set_id.value
381-
while True:
382-
if self._flag.value == 1:
383-
return True
384-
elif set_id != self._set_id.value:
385-
return True # flag is unset, but set_id changed, so there must have been a `set` followed by a `clear`
386-
# during `time.sleep()`. Fixes https://github.com/python/cpython/issues/95826
387-
elif timeout is not None and (time.monotonic() - start_time) > timeout:
388-
return False
352+
with self._cond:
353+
if self._flag.acquire(False):
354+
self._flag.release()
389355
else:
390-
# Fixes https://github.com/python/cpython/issues/85772 by spinning and sleeping.
391-
time.sleep(0.010) # sleep 10 milliseconds
356+
self._cond.wait(timeout)
357+
358+
if self._flag.acquire(False):
359+
self._flag.release()
360+
return True
361+
return False
392362

393363
def __repr__(self) -> str:
394364
set_status = 'set' if self.is_set() else 'unset'
395365
return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>"
396-
397366
#
398367
# Barrier
399368
#

‎Lib/signal.py‎

Lines changed: 64 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import _signal
2+
import os
23
from _signal import *
34
from enum import IntEnum as _IntEnum
5+
import threading
6+
import queue
47

58
_globals = globals()
69

@@ -42,6 +45,43 @@ def _enum_to_int(value):
4245
except (ValueError, TypeError):
4346
return value
4447

48+
_init_lock = threading.Lock()
49+
_signal_queue = queue.SimpleQueue() # SimpleQueue has reentrant put, so it can safely be called from signal handlers. https://github.com/python/cpython/issues/59181
50+
_bubble_queue = queue.SimpleQueue()
51+
_signal_thread = None
52+
_signo_to_handler = {}
53+
54+
def _signal_queue_handler():
55+
assert threading.current_thread() is not threading.main_thread()
56+
global _signal_queue, _signo_to_handler, _bubble_queue
57+
while True:
58+
(signo, stack_frame) = _signal_queue.get()
59+
try:
60+
handler = _signo_to_handler.get(signo, None)
61+
handler(signo, stack_frame)
62+
except Exception as e:
63+
_bubble_queue.put(e)
64+
# _signal.raise_signal(SIGTERM) # does not work when using event.wait()
65+
# _thread.interrupt_main(SIGTERM) # does not work when using event.wait()
66+
os.kill(os.getpid(), signo)
67+
68+
def _init_signal_thread():
69+
assert threading.current_thread() is threading.main_thread()
70+
global _signal_thread, _init_lock
71+
with _init_lock:
72+
if _signal_thread is None:
73+
_signal_thread = threading.Thread(target=_signal_queue_handler, daemon=True)
74+
_signal_thread.start()
75+
76+
def _push_signal_to_queue_handler(signo, stack_frame):
77+
assert threading.current_thread() is threading.main_thread()
78+
try:
79+
global _bubble_queue
80+
bubble_exception = _bubble_queue.get(block=False)
81+
raise bubble_exception
82+
except queue.Empty:
83+
global _signal_queue
84+
_signal_queue.put((signo, stack_frame))
4585

4686
# Similar to functools.wraps(), but only assign __doc__.
4787
# __module__ should be preserved,
@@ -53,16 +93,34 @@ def decorator(wrapper):
5393
return wrapper
5494
return decorator
5595

56-
@_wraps(_signal.signal)
57-
def signal(signalnum, handler):
58-
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
59-
return _int_to_enum(handler, Handlers)
96+
def signal(signalnum, handler, use_dedicated_thread=True):
97+
assert threading.current_thread() is threading.main_thread()
98+
global _signo_to_handler
99+
signal_int = _enum_to_int(signalnum)
100+
old_handler = _signo_to_handler.get(signal_int, None)
101+
if use_dedicated_thread and callable(handler):
102+
assert callable(handler)
103+
global _signal_thread
104+
if _signal_thread is None:
105+
_init_signal_thread()
106+
_signo_to_handler[signal_int] = handler
107+
handler = _signal.signal(signal_int, _enum_to_int(_push_signal_to_queue_handler))
108+
return old_handler or _int_to_enum(handler, Handlers)
109+
else:
110+
if signal_int in _signo_to_handler:
111+
del _signo_to_handler[signal_int]
112+
handler = _signal.signal(signal_int, _enum_to_int(handler))
113+
return old_handler or _int_to_enum(handler, Handlers)
60114

61115

62116
@_wraps(_signal.getsignal)
63117
def getsignal(signalnum):
64-
handler = _signal.getsignal(signalnum)
65-
return _int_to_enum(handler, Handlers)
118+
global _signo_to_handler
119+
if signalnum in _signo_to_handler:
120+
return _signo_to_handler[signalnum]
121+
else:
122+
handler = _signal.getsignal(signalnum)
123+
return _int_to_enum(handler, Handlers)
66124

67125

68126
if 'pthread_sigmask' in _globals:

0 commit comments

Comments
 (0)