Skip to content

Commit f077787

Browse files
only hold back bytes that can actually start a character
my first version read the lead byte and worked out the width by hand, which accepted lead bytes that are not legal. ff c0 c1 and f5 all looked like the start of something longer so they were held back and requeued every window, and the replacement character that used to appear right away stayed hidden until the process exited using an incremental decoder instead. it buffers only a real partial sequence and replaces anything that cannot begin a character straight away, and passing final when the provider is done closes it so a truncated tail is replaced too. it also knows about overlong forms and surrogates which my table did not added the invalid lead bytes as cases. four of the six fail on the old version
1 parent ea133bf commit f077787

2 files changed

Lines changed: 33 additions & 32 deletions

File tree

Lines changed: 13 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,14 @@
11
from __future__ import annotations
22

33
import asyncio
4+
import codecs
45
import time
56
from collections import deque
67
from collections.abc import Callable
78

89
from .pty_types import truncate_text_by_tokens
910

1011

11-
def _incomplete_utf8_suffix_len(data: bytes | bytearray) -> int:
12-
"""Length of a trailing UTF-8 sequence that is still waiting for its remaining bytes.
13-
14-
Returns 0 when the buffer does not end mid character.
15-
"""
16-
for back in range(1, min(4, len(data)) + 1):
17-
byte = data[-back]
18-
if byte < 0x80:
19-
return 0
20-
if byte >= 0xC0:
21-
if byte < 0xE0:
22-
expected = 2
23-
elif byte < 0xF0:
24-
expected = 3
25-
else:
26-
expected = 4
27-
return back if back < expected else 0
28-
return 0
29-
30-
3112
async def collect_pty_output(
3213
*,
3314
output_chunks: deque[bytes],
@@ -65,18 +46,18 @@ async def collect_pty_output(
6546
break
6647
output_notify.clear()
6748

68-
# PTY output is collected in repeated windows over one persistent deque, so a
69-
# character whose bytes straddle a window boundary would be replaced twice and
70-
# lost. Hand the incomplete tail back for the next window to finish. Once the
71-
# provider is done there is no next window, so the bytes are genuinely invalid.
72-
if not is_done():
73-
held_back = _incomplete_utf8_suffix_len(output)
74-
if held_back:
75-
tail = bytes(output[-held_back:])
76-
del output[-held_back:]
77-
async with output_lock:
78-
output_chunks.appendleft(tail)
49+
# Output is collected in repeated windows over one persistent deque, so a character
50+
# whose bytes straddle a window boundary used to be replaced twice and lost. An
51+
# incremental decoder keeps that trailing partial sequence instead of replacing it,
52+
# and it is handed back for the next window to finish. Bytes that cannot begin a
53+
# character are not held, they are replaced straight away as before. Completing the
54+
# decoder once the provider is done replaces a tail that no later window will finish.
55+
decoder = codecs.getincrementaldecoder("utf-8")("replace")
56+
text = decoder.decode(output, final=is_done())
57+
pending = decoder.getstate()[0]
58+
if pending:
59+
async with output_lock:
60+
output_chunks.appendleft(bytes(pending))
7961

80-
text = output.decode("utf-8", errors="replace")
8162
truncated, original_token_count = truncate_text_by_tokens(text, max_output_tokens)
8263
return truncated.encode("utf-8", errors="replace"), original_token_count

‎tests/sandbox/test_pty_output.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,23 @@ async def test_collect_pty_output_leaves_complete_multibyte_output_alone() -> No
133133
)
134134

135135
assert collected.decode("utf-8") == "héllo"
136+
137+
138+
# a lead byte that no character can start with, an overlong form, a value past the
139+
# end of the range, a lone continuation byte, and half of a surrogate pair
140+
@pytest.mark.asyncio
141+
@pytest.mark.parametrize("garbage", [b"\xff", b"\xc0", b"\xc1", b"\xf5", b"\x80", b"\xed\xa0\x80"])
142+
async def test_collect_pty_output_does_not_hold_back_bytes_that_start_no_character(
143+
garbage: bytes,
144+
) -> None:
145+
# the process is still running, but these bytes will never be completed by a later
146+
# window, so holding them would hide the output until it exits
147+
chunks: deque[bytes] = deque([b"ok " + garbage])
148+
lock = asyncio.Lock()
149+
notify = asyncio.Event()
150+
done = {"value": False}
151+
152+
collected = await _one_window(chunks, lock, notify, done)
153+
154+
assert collected.decode("utf-8").startswith("ok �")
155+
assert not chunks

0 commit comments

Comments
 (0)