Skip to content

Commit 6bb7079

Browse files
PR Batch Testerclaude
andcommitted
fix(shared): size frame parts by their UTF-8 bytes
Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7637a45 commit 6bb7079

2 files changed

Lines changed: 26 additions & 19 deletions

File tree

‎packages/shared/src/socketFrameParts.test.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import * as Socket from "effect/socket/Socket";
44

55
import {
66
joinSocketFrameParts,
7-
SOCKET_FRAME_PART_LENGTH,
7+
SOCKET_FRAME_PART_BYTES,
88
splitLargeSocketFrames,
99
} from "./socketFrameParts.ts";
1010

@@ -38,10 +38,11 @@ function replayingSocket(batches: ReadonlyArray<readonly [string, ...Array<strin
3838

3939
it.effect("a large message reaches the client whole, with a frame for every part on the way", () =>
4040
Effect.gen(function* () {
41-
// Emoji put surrogate pairs across part boundaries.
41+
// Two-, three- and four-byte characters, the last as surrogate pairs that
42+
// fall across part boundaries.
4243
const message = JSON.stringify({
4344
_tag: "Chunk",
44-
values: ["😀".repeat(SOCKET_FRAME_PART_LENGTH), "x".repeat(SOCKET_FRAME_PART_LENGTH)],
45+
values: ["é€😀x".repeat(SOCKET_FRAME_PART_BYTES / 4)],
4546
});
4647
const sent: Array<string | Uint8Array> = [];
4748
const serverWriter = yield* splitLargeSocketFrames(recordingSocket(sent)).writer;
@@ -50,9 +51,10 @@ it.effect("a large message reaches the client whole, with a frame for every part
5051

5152
const parts = sent.filter((frame) => typeof frame === "string");
5253
assert.isAbove(parts.length, 3);
53-
assert.isTrue(parts.every((part) => part.length <= SOCKET_FRAME_PART_LENGTH + 1));
54+
const encoded = parts.map((part) => new TextEncoder().encode(part));
55+
assert.isTrue(encoded.every((bytes) => bytes.length <= SOCKET_FRAME_PART_BYTES));
5456
assert.isTrue(
55-
parts.every((part) => new TextDecoder().decode(new TextEncoder().encode(part)) === part),
57+
encoded.every((bytes, index) => new TextDecoder().decode(bytes) === parts[index]),
5658
);
5759

5860
const [first, ...rest] = parts;
@@ -71,7 +73,7 @@ it.effect("a server that predates parts still reaches the client whole", () =>
7173
Effect.gen(function* () {
7274
const message = JSON.stringify({
7375
_tag: "Chunk",
74-
values: ["x".repeat(3 * SOCKET_FRAME_PART_LENGTH)],
76+
values: ["x".repeat(3 * SOCKET_FRAME_PART_BYTES)],
7577
});
7678
const reader = yield* joinSocketFrameParts(replayingSocket([[message]])).reader;
7779
assert.deepStrictEqual(yield* reader.pull, [message]);

‎packages/shared/src/socketFrameParts.ts‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ export const SOCKET_FRAME_PARTS_QUERY_PARAM = "frameParts";
1313
// link a multi-megabyte config snapshot outlasts the ping timeout, so the
1414
// client drops a healthy server and asks for the same snapshot again. Parts
1515
// this size keep a frame arriving within the timeout down to a few KB/s.
16-
export const SOCKET_FRAME_PART_LENGTH = 64 * 1024;
16+
export const SOCKET_FRAME_PART_BYTES = 64 * 1024;
1717

1818
// No JSON message starts with a record separator. It marks every part but the last.
1919
const PART_PREFIX = "\u001e";
@@ -22,20 +22,27 @@ const PART_PREFIX = "\u001e";
2222
// ignores pongs, so each held-back part reads to it as one.
2323
const PART_RECEIVED_FRAME = '{"_tag":"Pong"}';
2424

25+
/** Splits on the UTF-8 size the frame has on the wire, never inside a surrogate pair. */
2526
export function splitSocketFrame(
2627
frame: string,
27-
partLength = SOCKET_FRAME_PART_LENGTH,
28+
partBytes = SOCKET_FRAME_PART_BYTES,
2829
): Arr.NonEmptyReadonlyArray<string> {
29-
if (frame.length <= partLength) return [frame];
30+
// UTF-8 spends at most 3 bytes on a UTF-16 code unit.
31+
if (frame.length * 3 <= partBytes) return [frame];
3032
const parts: Array<string> = [];
3133
let start = 0;
32-
while (frame.length - start > partLength) {
33-
let end = start + partLength;
34-
// A lone surrogate half does not survive the UTF-8 encoding of a text frame.
35-
const last = frame.charCodeAt(end - 1);
36-
if (last >= 0xd800 && last <= 0xdbff) end -= 1;
37-
parts.push(PART_PREFIX + frame.slice(start, end));
38-
start = end;
34+
let bytes = PART_PREFIX.length;
35+
for (let index = 0; index < frame.length;) {
36+
const code = frame.charCodeAt(index);
37+
const pair = code >= 0xd800 && code <= 0xdbff && index + 1 < frame.length;
38+
const size = pair ? 4 : code < 0x80 ? 1 : code < 0x800 ? 2 : 3;
39+
if (bytes + size > partBytes) {
40+
parts.push(PART_PREFIX + frame.slice(start, index));
41+
start = index;
42+
bytes = PART_PREFIX.length;
43+
}
44+
bytes += size;
45+
index += pair ? 2 : 1;
3946
}
4047
return Arr.append(parts, frame.slice(start));
4148
}
@@ -48,9 +55,7 @@ export function splitLargeSocketFrames(socket: Socket.Socket): Socket.Socket {
4855
reader: socket.reader,
4956
writer: Effect.map(socket.writer, (writer) => ({
5057
write: (chunk) =>
51-
typeof chunk === "string" && chunk.length > SOCKET_FRAME_PART_LENGTH
52-
? writer.writeAll(splitSocketFrame(chunk))
53-
: writer.write(chunk),
58+
typeof chunk === "string" ? writer.writeAll(splitSocketFrame(chunk)) : writer.write(chunk),
5459
writeAll: (chunks) => writer.writeAll(Arr.flatMap(chunks, split)),
5560
})),
5661
});

0 commit comments

Comments
 (0)