Skip to content

Commit ae43e63

Browse files
committed
fix: use poll() instead of epoll() for waiting
epoll() hangs when the fd it listens to is closed. poll() doesn't have this problem (as a consequence, hanging only happened in the Python code, as wait_c is poll-based).
1 parent 8b564e8 commit ae43e63

4 files changed

Lines changed: 73 additions & 15 deletions

File tree

‎docs/news.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
Psycopg 3.1.12 (unreleased)
1111
^^^^^^^^^^^^^^^^^^^^^^^^^^^
1212

13-
- Fix hanging if an async connection is closed while querying (:ticket:`#608`).
13+
- Fix possible hanging if a connection is closed while querying (:ticket:`#608`).
1414
- Fix memory leak when `~register_*()` functions are called repeatedly
1515
(:ticket:`#647`).
1616

‎psycopg/psycopg/waiting.py‎

Lines changed: 69 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import sys
1414
import select
1515
import selectors
16-
from typing import Dict, Optional
16+
from typing import Optional
1717
from asyncio import get_event_loop, wait_for, Event, TimeoutError
1818
from selectors import DefaultSelector
1919

@@ -218,6 +218,8 @@ def wakeup(state: Ready) -> None:
218218
def wait_select(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV:
219219
"""
220220
Wait for a generator using select where supported.
221+
222+
BUG: on Linux, can't select on FD >= 1024. On Windows it's fine.
221223
"""
222224
try:
223225
s = next(gen)
@@ -246,16 +248,15 @@ def wait_select(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) ->
246248
return rv
247249

248250

249-
poll_evmasks: Dict[Wait, int]
250-
251251
if hasattr(selectors, "EpollSelector"):
252-
poll_evmasks = {
253-
WAIT_R: select.EPOLLONESHOT | select.EPOLLIN,
254-
WAIT_W: select.EPOLLONESHOT | select.EPOLLOUT,
255-
WAIT_RW: select.EPOLLONESHOT | select.EPOLLIN | select.EPOLLOUT,
252+
_epoll_evmasks = {
253+
WAIT_R: select.EPOLLONESHOT | select.EPOLLIN | select.EPOLLERR,
254+
WAIT_W: select.EPOLLONESHOT | select.EPOLLOUT | select.EPOLLERR,
255+
WAIT_RW: select.EPOLLONESHOT
256+
| (select.EPOLLIN | select.EPOLLOUT | select.EPOLLERR),
256257
}
257258
else:
258-
poll_evmasks = {}
259+
_epoll_evmasks = {}
259260

260261

261262
def wait_epoll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV:
@@ -266,6 +267,13 @@ def wait_epoll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) ->
266267
strategy is `epoll` then this function will be used instead of `wait`.
267268
268269
See also: https://linux.die.net/man/2/epoll_ctl
270+
271+
BUG: if the connection FD is closed, `epoll.poll()` hangs. Same for
272+
EpollSelector. For this reason, wait_poll() is currently preferable.
273+
To reproduce the bug:
274+
275+
export PSYCOPG_WAIT_FUNC=wait_epoll
276+
pytest tests/test_concurrency.py::test_concurrent_close
269277
"""
270278
try:
271279
s = next(gen)
@@ -276,7 +284,7 @@ def wait_epoll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) ->
276284
timeout = int(timeout * 1000.0)
277285

278286
with select.epoll() as epoll:
279-
evmask = poll_evmasks[s]
287+
evmask = _epoll_evmasks[s]
280288
epoll.register(fileno, evmask)
281289
while True:
282290
fileevs = None
@@ -290,14 +298,61 @@ def wait_epoll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) ->
290298
ready |= READY_W
291299
# assert s & ready
292300
s = gen.send(ready)
293-
evmask = poll_evmasks[s]
301+
evmask = _epoll_evmasks[s]
294302
epoll.modify(fileno, evmask)
295303

296304
except StopIteration as ex:
297305
rv: RV = ex.args[0] if ex.args else None
298306
return rv
299307

300308

309+
if hasattr(selectors, "PollSelector"):
310+
_poll_evmasks = {
311+
WAIT_R: select.POLLIN,
312+
WAIT_W: select.POLLOUT,
313+
WAIT_RW: select.POLLIN | select.POLLOUT,
314+
}
315+
else:
316+
_poll_evmasks = {}
317+
318+
319+
def wait_poll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) -> RV:
320+
"""
321+
Wait for a generator using poll where supported.
322+
323+
Parameters are like for `wait()`.
324+
"""
325+
try:
326+
s = next(gen)
327+
328+
if timeout is None or timeout < 0:
329+
timeout = 0
330+
else:
331+
timeout = int(timeout * 1000.0)
332+
333+
poll = select.poll()
334+
evmask = _poll_evmasks[s]
335+
poll.register(fileno, evmask)
336+
while True:
337+
fileevs = None
338+
while not fileevs:
339+
fileevs = poll.poll(timeout)
340+
ev = fileevs[0][1]
341+
ready = 0
342+
if ev & ~select.POLLOUT:
343+
ready = READY_R
344+
if ev & ~select.POLLIN:
345+
ready |= READY_W
346+
# assert s & ready
347+
s = gen.send(ready)
348+
evmask = _poll_evmasks[s]
349+
poll.modify(fileno, evmask)
350+
351+
except StopIteration as ex:
352+
rv: RV = ex.args[0] if ex.args else None
353+
return rv
354+
355+
301356
if _psycopg:
302357
wait_c = _psycopg.wait_c
303358

@@ -329,8 +384,10 @@ def wait_epoll(gen: PQGen[RV], fileno: int, timeout: Optional[float] = None) ->
329384
# On Windows, SelectSelector should be the default.
330385
wait = wait_select
331386

332-
elif selectors.DefaultSelector is getattr(selectors, "EpollSelector", None):
333-
wait = wait_epoll
387+
elif hasattr(selectors, "PollSelector"):
388+
# On linux, EpollSelector is the default. However, it hangs if the fd is
389+
# closed while polling.
390+
wait = wait_poll
334391

335392
else:
336393
wait = wait_selector

‎tests/test_concurrency.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -401,8 +401,8 @@ def test(arg):
401401
reason="Fails with: An operation was attempted on something that is not a socket",
402402
)
403403
def test_concurrent_close(dsn, conn):
404-
# Verify something similar to the problem in #608, which doesn't affect
405-
# sync connections anyway.
404+
# Test issue #608: concurrent closing shouldn't hang the server
405+
# (although, at the moment, it doesn't cancel a running query).
406406
pid = conn.info.backend_pid
407407
conn.autocommit = True
408408

‎tests/test_waiting.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
pytest.param(
2323
"wait_epoll", marks=pytest.mark.skipif("not hasattr(select, 'epoll')")
2424
),
25+
pytest.param("wait_poll", marks=pytest.mark.skipif("not hasattr(select, 'poll')")),
2526
pytest.param("wait_c", marks=pytest.mark.skipif("not psycopg._cmodule._psycopg")),
2627
]
2728

0 commit comments

Comments
 (0)