Skip to content

Commit 52ba328

Browse files
fix(server): avoid reparsing streamed replies from the beginning (#24432)
1 parent d27d03b commit 52ba328

2 files changed

Lines changed: 89 additions & 17 deletions

File tree

‎apps/server/src/orchestration-v2/assistantStreaming.test.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { MessageId, ProviderDriverKind, ThreadId } from "@t3tools/contracts";
22
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
33
import * as DateTime from "effect/DateTime";
4-
import { assert, describe, expect, it } from "vite-plus/test";
4+
import { assert, describe, expect, it, vi } from "vite-plus/test";
55
import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts";
66

77
describe("V2 assistant streaming", () => {
@@ -173,6 +173,13 @@ it.each<{ readonly name: string; readonly mode?: "turn"; readonly steps: Readonl
173173
{ idle: 4_800, shows: [""] },
174174
],
175175
},
176+
{
177+
name: "a rewrite that drops an open code fence",
178+
steps: [
179+
{ at: 0, text: "Intro\n\n```ts\nx()\n", shows: "Intro\n\n" },
180+
{ at: 400, text: "Retry\n\n- a\n- b", shows: "Retry\n\n- a\n" },
181+
],
182+
},
176183
{
177184
name: "turn mode",
178185
mode: "turn",
@@ -193,3 +200,22 @@ it.each<{ readonly name: string; readonly mode?: "turn"; readonly steps: Readonl
193200
);
194201
}
195202
});
203+
204+
it("scans each streamed line once while a long code fence is still open", () => {
205+
const filter = makeAssistantStreamingFilter("paragraph", 800);
206+
const lineCount = 400;
207+
let text = "Intro\n\n```ts\n";
208+
const indexOf = vi.spyOn(String.prototype, "indexOf");
209+
try {
210+
for (let line = 0; line < lineCount; line++) {
211+
text += `const value${line} = ${line};\n`;
212+
filter.filter(snapshot("m", text, true), line * 400);
213+
}
214+
const newlineSearches = indexOf.mock.calls.filter(([search]) => search === "\n").length;
215+
expect(newlineSearches).toBeLessThan(lineCount * 4);
216+
} finally {
217+
indexOf.mockRestore();
218+
}
219+
text += "```\n";
220+
assert.equal(shown(filter.filter(snapshot("m", text, true), lineCount * 400)), text);
221+
});

‎apps/server/src/orchestration-v2/assistantStreaming.ts‎

Lines changed: 62 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,44 @@ const SECTION_TITLE_PATTERN = /^ {0,3}(?:#{1,6}(?:[ \t]|$)|\*\*(?:[^*]|\*(?!\*))
1111
// Unindented ATX headings end preceding paragraphs and lists without blank lines; bold lines do not.
1212
const TOP_LEVEL_HEADING_PATTERN = /^#{1,6}(?:[ \t]|$)/;
1313

14-
/** Emit terminated lines only, except partial list-start lookahead for tight lists. Hold section titles until content follows. */
15-
export function splitBufferedAssistantText(text: string): { ready: string; rest: string } {
16-
let openFence: { marker: string; indent: number } | null = null;
17-
let boundary = -1;
18-
let lineStart = 0;
19-
let titleAwaitingContent = false;
14+
// Matches LIST_ITEM_START_PATTERN on the trimmed partial line without copying it.
15+
const PARTIAL_LIST_ITEM_START_PATTERN = /[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t][ \t\r]*[^ \t\r]/y;
16+
17+
interface AssistantTextScan {
18+
readonly text: string;
19+
// Start of the partial line; every line before it has been scanned.
20+
readonly lineStart: number;
21+
readonly openFence: { readonly marker: string; readonly indent: number } | null;
22+
readonly boundary: number;
23+
readonly titleAwaitingContent: boolean;
24+
}
25+
26+
const EMPTY_SCAN: AssistantTextScan = {
27+
text: "",
28+
lineStart: 0,
29+
openFence: null,
30+
boundary: -1,
31+
titleAwaitingContent: false,
32+
};
33+
34+
function trimLineEnd(text: string, start: number, end: number): string {
35+
while (end > start) {
36+
const char = text.charCodeAt(end - 1);
37+
if (char !== 32 && char !== 9 && char !== 13) break;
38+
end--;
39+
}
40+
return text.slice(start, end);
41+
}
42+
43+
/** Scans the terminated lines of `text`, resuming after `previous` when `text` extends it. */
44+
function advanceScan(previous: AssistantTextScan, text: string): AssistantTextScan {
45+
const resumed = text.startsWith(previous.text);
46+
let { openFence, boundary, lineStart, titleAwaitingContent } = resumed ? previous : EMPTY_SCAN;
47+
let searchFrom = resumed ? previous.text.length : 0;
2048
for (;;) {
21-
const newline = text.indexOf("\n", lineStart);
22-
const line = text
23-
.slice(lineStart, newline === -1 ? text.length : newline)
24-
.replace(/[ \t\r]+$/, "");
49+
const newline = text.indexOf("\n", searchFrom);
50+
if (newline === -1) break;
51+
const line = trimLineEnd(text, lineStart, newline);
2552
if (
2653
openFence === null &&
2754
lineStart > 0 &&
@@ -30,9 +57,6 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest:
3057
) {
3158
boundary = lineStart;
3259
}
33-
if (newline === -1) {
34-
break;
35-
}
3660
const fenceMatch = MARKDOWN_FENCE_PATTERN.exec(line);
3761
if (fenceMatch) {
3862
const indent = fenceMatch[1]!.length;
@@ -61,7 +85,23 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest:
6185
titleAwaitingContent = SECTION_TITLE_PATTERN.test(line);
6286
}
6387
lineStart = newline + 1;
88+
searchFrom = lineStart;
89+
}
90+
return { text, lineStart, openFence, boundary, titleAwaitingContent };
91+
}
92+
93+
// The partial line may still change, so its list item start is checked here, not kept in the scan.
94+
function readyBoundary(scan: AssistantTextScan): number {
95+
if (scan.openFence === null && scan.lineStart > 0 && !scan.titleAwaitingContent) {
96+
PARTIAL_LIST_ITEM_START_PATTERN.lastIndex = scan.lineStart;
97+
if (PARTIAL_LIST_ITEM_START_PATTERN.test(scan.text)) return scan.lineStart;
6498
}
99+
return scan.boundary;
100+
}
101+
102+
/** Emit terminated lines only, except partial list-start lookahead for tight lists. Hold section titles until content follows. */
103+
export function splitBufferedAssistantText(text: string): { ready: string; rest: string } {
104+
const boundary = readyBoundary(advanceScan(EMPTY_SCAN, text));
65105
if (boundary === -1) {
66106
return { ready: "", rest: text };
67107
}
@@ -84,6 +124,7 @@ interface StreamTrack {
84124
// Latest snapshot whose text has not all been delivered yet.
85125
readonly held: ProviderAdapter.ProviderAdapterV2Event | null;
86126
readonly dueAt: number;
127+
readonly scan: AssistantTextScan;
87128
}
88129

89130
function streamedMessage(event: ProviderAdapter.ProviderAdapterV2Event): StreamedMessage | null {
@@ -148,25 +189,29 @@ export function makeAssistantStreamingFilter(
148189
if (mode === "turn") return null;
149190
const previous = tracks.get(key);
150191
const delivered = previous?.text ?? "";
151-
const hold = () => {
192+
const hold = (scan = previous?.scan ?? EMPTY_SCAN) => {
152193
track(key, {
153194
text: delivered,
154195
at: previous?.at ?? Number.NEGATIVE_INFINITY,
155196
held: message.text !== delivered ? event : null,
156197
dueAt: now + idleFlushMs,
198+
scan,
157199
});
158200
return null;
159201
};
160202
if (previous && now - previous.at < 400) return hold();
161-
const { ready } = splitBufferedAssistantText(message.text);
203+
const scan = advanceScan(previous?.scan ?? EMPTY_SCAN, message.text);
204+
const boundary = readyBoundary(scan);
205+
const ready = boundary === -1 ? "" : message.text.slice(0, boundary);
162206
// An idle delivery may already show more than the next boundary; never shrink it. A
163207
// provider rewrite or truncation is not a prefix and replaces it.
164-
if (delivered.startsWith(ready)) return hold();
208+
if (delivered.startsWith(ready)) return hold(scan);
165209
track(key, {
166210
text: ready,
167211
at: now,
168212
held: ready !== message.text ? event : null,
169213
dueAt: now + idleFlushMs,
214+
scan,
170215
});
171216
return withText(event, ready);
172217
},
@@ -188,6 +233,7 @@ export function makeAssistantStreamingFilter(
188233
at: now,
189234
held: null,
190235
dueAt: now,
236+
scan: current.scan,
191237
});
192238
}
193239
return due;

0 commit comments

Comments
 (0)