Skip to content

Commit 576892e

Browse files
Fixed certain pending tasks not woken up by the closing of memory object streams (#1443)
Fixes #1391. Closes #1389. Closes #1415. --------- Co-authored-by: David Brochart <[email protected]>
1 parent 02ff4f8 commit 576892e

3 files changed

Lines changed: 127 additions & 6 deletions

File tree

‎changelog.d/1391.fixed.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fixed ``close()`` on memory object streams not waking up tasks blocked in ``send()``
2+
or ``receive()`` on the same stream

‎src/anyio/streams/memory.py‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ def statistics(self) -> MemoryObjectStreamStatistics:
8080
class MemoryObjectReceiveStream(ObjectReceiveStream[T_co], Generic[T_co]):
8181
_state: _MemoryObjectStreamState[T_co]
8282
_closed: bool = field(init=False, default=False)
83+
_waiters: set[Event] | None = field(init=False, default=None)
8384

8485
def __post_init__(self) -> None:
8586
self._state.open_receive_channels += 1
@@ -117,20 +118,29 @@ async def receive(self) -> T_co:
117118
try:
118119
return self.receive_nowait()
119120
except WouldBlock:
121+
if self._waiters is None:
122+
self._waiters = set[Event]()
123+
120124
# Add ourselves in the queue
121125
receive_event = Event()
126+
self._waiters.add(receive_event)
122127
receiver = _MemoryObjectItemReceiver[T_co]()
123128
self._state.waiting_receivers[receive_event] = receiver
124129

125130
try:
126131
await receive_event.wait()
127132
finally:
128133
self._state.waiting_receivers.pop(receive_event, None)
134+
if self._waiters is not None:
135+
self._waiters.remove(receive_event)
129136

130137
try:
131138
return receiver.item
132139
except AttributeError:
133-
raise EndOfStream from None
140+
if self._closed:
141+
raise ClosedResourceError from None
142+
else:
143+
raise EndOfStream from None
134144

135145
def clone(self) -> MemoryObjectReceiveStream[T_co]:
136146
"""
@@ -163,6 +173,13 @@ def close(self) -> None:
163173
for event in send_events:
164174
event.set()
165175

176+
if self._waiters is not None:
177+
for event in self._waiters:
178+
self._state.waiting_receivers.pop(event, None)
179+
event.set()
180+
181+
self._waiters = None
182+
166183
async def aclose(self) -> None:
167184
self.close()
168185

@@ -199,6 +216,7 @@ def __del__(self) -> None:
199216
class MemoryObjectSendStream(ObjectSendStream[T_contra], Generic[T_contra]):
200217
_state: _MemoryObjectStreamState[T_contra]
201218
_closed: bool = field(init=False, default=False)
219+
_waiters: set[Event] | None = field(init=False, default=None)
202220

203221
def __post_init__(self) -> None:
204222
self._state.open_send_channels += 1
@@ -249,18 +267,26 @@ async def send(self, item: T_contra) -> None:
249267
try:
250268
self.send_nowait(item)
251269
except WouldBlock:
270+
if self._waiters is None:
271+
self._waiters = set[Event]()
272+
252273
# Wait until there's someone on the receiving end
253274
send_event = Event()
275+
self._waiters.add(send_event)
254276
self._state.waiting_senders[send_event] = item
255277
try:
256278
await send_event.wait()
257-
except BaseException:
258-
self._state.waiting_senders.pop(send_event, None)
259-
raise
279+
finally:
280+
if item_not_consumed := send_event in self._state.waiting_senders:
281+
del self._state.waiting_senders[send_event]
260282

261-
if send_event in self._state.waiting_senders:
262-
del self._state.waiting_senders[send_event]
283+
withdrawn = self._closed and send_event in self._waiters
284+
self._waiters.discard(send_event)
285+
286+
if item_not_consumed:
263287
raise BrokenResourceError from None
288+
elif withdrawn:
289+
raise ClosedResourceError from None
264290

265291
def clone(self) -> MemoryObjectSendStream[T_contra]:
266292
"""
@@ -294,6 +320,19 @@ def close(self) -> None:
294320
for event in receive_events:
295321
event.set()
296322

323+
if self._waiters is not None:
324+
withdrawn = set[Event]()
325+
for event in self._waiters:
326+
if event in self._state.waiting_senders:
327+
del self._state.waiting_senders[event]
328+
withdrawn.add(event)
329+
330+
event.set()
331+
332+
# An event remaining on this set signals that its item was never
333+
# delivered, and send() will raise ClosedResourceError
334+
self._waiters = withdrawn
335+
297336
async def aclose(self) -> None:
298337
self.close()
299338

‎tests/streams/test_memory.py‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,38 @@ async def test_receive_send_closed_receive_stream() -> None:
137137
await send.send(None)
138138

139139

140+
async def test_close_receive_clone_while_receiving() -> None:
141+
send, receive1 = create_memory_object_stream[str](1)
142+
receive2 = receive1.clone()
143+
with send, receive2:
144+
with pytest.RaisesGroup(ClosedResourceError):
145+
async with create_task_group() as tg:
146+
tg.start_soon(receive1.receive)
147+
await wait_all_tasks_blocked()
148+
# Send before the woken receiver task gets a chance to run
149+
receive1.close()
150+
send.send_nowait("hello")
151+
152+
assert receive2.receive_nowait() == "hello"
153+
154+
155+
async def test_close_send_clone_while_sending() -> None:
156+
send1, receive = create_memory_object_stream[str]()
157+
send2 = send1.clone()
158+
with send1, send2, receive:
159+
with pytest.RaisesGroup(ClosedResourceError):
160+
async with create_task_group() as tg:
161+
tg.start_soon(send1.send, "withdrawn")
162+
await wait_all_tasks_blocked()
163+
send1.close()
164+
with pytest.raises(WouldBlock):
165+
receive.receive_nowait()
166+
167+
async with create_task_group() as tg:
168+
tg.start_soon(send2.send, "hello")
169+
assert await receive.receive() == "hello"
170+
171+
140172
async def test_cancel_receive() -> None:
141173
send, receive = create_memory_object_stream[str]()
142174
with send, receive:
@@ -189,6 +221,25 @@ async def test_close_send_while_receiving() -> None:
189221
await send.aclose()
190222

191223

224+
async def test_close_send_while_sending() -> None:
225+
send, receive = create_memory_object_stream[None]()
226+
with receive, pytest.RaisesGroup(ClosedResourceError, EndOfStream):
227+
async with create_task_group() as tg:
228+
tg.start_soon(send.send, None)
229+
await wait_all_tasks_blocked()
230+
send.close()
231+
receive.receive_nowait()
232+
233+
234+
async def test_close_receive_while_receiving() -> None:
235+
send, receive = create_memory_object_stream[NoReturn](1)
236+
with send, pytest.RaisesGroup(ClosedResourceError):
237+
async with create_task_group() as tg:
238+
tg.start_soon(receive.receive)
239+
await wait_all_tasks_blocked()
240+
await receive.aclose()
241+
242+
192243
async def test_close_receive_while_sending() -> None:
193244
# We send None here as a regression test for #731
194245
send, receive = create_memory_object_stream[None](0)
@@ -199,6 +250,35 @@ async def test_close_receive_while_sending() -> None:
199250
await receive.aclose()
200251

201252

253+
async def test_close_send_after_item_received() -> None:
254+
send, receive = create_memory_object_stream[None]()
255+
with receive:
256+
async with create_task_group() as tg:
257+
tg.start_soon(send.send, None)
258+
await wait_all_tasks_blocked()
259+
# Close before the woken sender task gets a chance to run
260+
assert receive.receive_nowait() is None
261+
send.close()
262+
263+
264+
async def test_close_receive_after_item_sent() -> None:
265+
received: list[str] = []
266+
267+
async def receiver() -> None:
268+
received.append(await receive.receive())
269+
270+
send, receive = create_memory_object_stream[str]()
271+
with send:
272+
async with create_task_group() as tg:
273+
tg.start_soon(receiver)
274+
await wait_all_tasks_blocked()
275+
# Close before the woken receiver task gets a chance to run
276+
send.send_nowait("hello")
277+
receive.close()
278+
279+
assert received == ["hello"]
280+
281+
202282
async def test_receive_after_send_closed() -> None:
203283
send, receive = create_memory_object_stream[str](1)
204284
with receive:

0 commit comments

Comments
 (0)