Skip to content

Commit 1bbca0e

Browse files
t3dotggclaude
andauthored
feat(settings): choose how responses stream, with a warning on legacy token mode (#11678)
Co-authored-by: Claude Fable 5.1 <[email protected]>
1 parent 3b75e60 commit 1bbca0e

7 files changed

Lines changed: 252 additions & 81 deletions

File tree

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.test.ts‎

Lines changed: 64 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -465,11 +465,11 @@ describe("ProviderRuntimeIngestion", () => {
465465
});
466466

467467
it.each([
468-
{ delivery: "buffered", enableLegacyTokenStreaming: false },
469-
{ delivery: "streamed", enableLegacyTokenStreaming: true },
468+
{ delivery: "buffered", responseStreamingMode: "paragraph" as const },
469+
{ delivery: "streamed", responseStreamingMode: "token" as const },
470470
])("settles OpenCode aborted turns and saves $delivery assistant text", async (settings) => {
471471
const harness = await createHarness({
472-
serverSettings: { enableLegacyTokenStreaming: settings.enableLegacyTokenStreaming },
472+
serverSettings: { responseStreamingMode: settings.responseStreamingMode },
473473
});
474474
const threadId = asThreadId("thread-1");
475475
const turnId = asTurnId("opencode-aborted-turn");
@@ -521,7 +521,7 @@ describe("ProviderRuntimeIngestion", () => {
521521
"finalizes old buffered text on late %s without stopping the newer turn",
522522
async (terminalType) => {
523523
const harness = await createHarness({
524-
serverSettings: { enableLegacyTokenStreaming: false },
524+
serverSettings: { responseStreamingMode: "paragraph" },
525525
});
526526
const threadId = asThreadId("thread-1");
527527
const oldTurnId = asTurnId("old-buffered-turn");
@@ -605,7 +605,7 @@ describe("ProviderRuntimeIngestion", () => {
605605
{ source: "an unspecified turn", turnId: undefined },
606606
])("ignores late OpenCode aborts for $source across newer turns", async (lateAbort) => {
607607
const harness = await createHarness({
608-
serverSettings: { enableLegacyTokenStreaming: true },
608+
serverSettings: { responseStreamingMode: "token" },
609609
});
610610
const threadId = asThreadId("thread-1");
611611
const stoppedTurnId = asTurnId("opencode-stopped-turn");
@@ -2712,7 +2712,7 @@ describe("ProviderRuntimeIngestion", () => {
27122712
});
27132713

27142714
it("keeps streaming while an async question is pending", async () => {
2715-
const harness = await createHarness({ serverSettings: { enableLegacyTokenStreaming: true } });
2715+
const harness = await createHarness({ serverSettings: { responseStreamingMode: "token" } });
27162716
const base = {
27172717
provider: ProviderDriverKind.make("codex"),
27182718
createdAt: "2026-01-01T00:00:00.000Z",
@@ -2954,7 +2954,7 @@ describe("ProviderRuntimeIngestion", () => {
29542954
});
29552955

29562956
it("starts a new streaming assistant message segment after approval", async () => {
2957-
const harness = await createHarness({ serverSettings: { enableLegacyTokenStreaming: true } });
2957+
const harness = await createHarness({ serverSettings: { responseStreamingMode: "token" } });
29582958
const startedAt = "2026-03-28T07:00:00.000Z";
29592959
const pausedAt = "2026-03-28T07:00:01.000Z";
29602960
const resumedAt = "2026-03-28T07:00:02.000Z";
@@ -3061,7 +3061,7 @@ describe("ProviderRuntimeIngestion", () => {
30613061
});
30623062

30633063
it("streams assistant deltas when thread.turn.start requests streaming mode", async () => {
3064-
const harness = await createHarness({ serverSettings: { enableLegacyTokenStreaming: true } });
3064+
const harness = await createHarness({ serverSettings: { responseStreamingMode: "token" } });
30653065
const now = "2026-01-01T00:00:00.000Z";
30663066

30673067
await Effect.runPromise(
@@ -3235,6 +3235,62 @@ describe("ProviderRuntimeIngestion", () => {
32353235
);
32363236
});
32373237

3238+
it("holds every paragraph until completion in turn mode", async () => {
3239+
const harness = await createHarness({ serverSettings: { responseStreamingMode: "turn" } });
3240+
const now = "2026-01-01T00:00:00.000Z";
3241+
const codex = ProviderDriverKind.make("codex");
3242+
const threadId = asThreadId("thread-1");
3243+
const turnId = asTurnId("turn-wait-mode");
3244+
const itemId = asItemId("item-wait-mode");
3245+
3246+
await harness.emitAndDrain([
3247+
{
3248+
type: "turn.started",
3249+
eventId: asEventId("evt-wait-started"),
3250+
provider: codex,
3251+
createdAt: now,
3252+
threadId,
3253+
turnId,
3254+
},
3255+
]);
3256+
harness.advanceClock(1_000);
3257+
await harness.emitAndDrain([
3258+
{
3259+
type: "content.delta",
3260+
eventId: asEventId("evt-wait-delta"),
3261+
provider: codex,
3262+
createdAt: now,
3263+
threadId,
3264+
turnId,
3265+
itemId,
3266+
payload: {
3267+
streamKind: "assistant_text",
3268+
delta: "First paragraph.\n\nSecond paragraph.\n\n",
3269+
},
3270+
},
3271+
]);
3272+
const messageText = async () =>
3273+
(await harness.readModel()).threads
3274+
.find((t) => t.id === threadId)
3275+
?.messages.find((m: ProviderRuntimeTestMessage) => m.id === `assistant:${itemId}`)?.text;
3276+
// Paragraph mode would have delivered both paragraphs by now.
3277+
expect(await messageText()).toBeUndefined();
3278+
3279+
await harness.emitAndDrain([
3280+
{
3281+
type: "item.completed",
3282+
eventId: asEventId("evt-wait-completed"),
3283+
provider: codex,
3284+
createdAt: now,
3285+
threadId,
3286+
turnId,
3287+
itemId,
3288+
payload: { itemType: "assistant_message", status: "completed" },
3289+
},
3290+
]);
3291+
expect(await messageText()).toBe("First paragraph.\n\nSecond paragraph.\n\n");
3292+
});
3293+
32383294
it("holds paragraphs that finish inside the pacing window and lands them together", async () => {
32393295
const harness = await createHarness();
32403296
const codex = ProviderDriverKind.make("codex");

‎apps/server/src/orchestration/Layers/ProviderRuntimeIngestion.ts‎

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import {
22
ApprovalRequestId,
3-
type AssistantDeliveryMode,
43
CommandId,
54
MessageId,
65
type OrchestrationEvent,
@@ -14,7 +13,9 @@ import {
1413
TurnId,
1514
type OrchestrationCheckpointSummary,
1615
type OrchestrationThreadActivity,
16+
type ProjectId,
1717
type ProviderRuntimeEvent,
18+
type ResponseStreamingMode,
1819
RuntimeRequestId,
1920
} from "@t3tools/contracts";
2021
import * as Cache from "effect/Cache";
@@ -1167,7 +1168,19 @@ const make = Effect.gen(function* () {
11671168
});
11681169
});
11691170

1170-
const appendBufferedAssistantText = (messageId: MessageId, delta: string, atMillis: number) =>
1171+
const resolveResponseStreamingMode = (projectId: ProjectId) =>
1172+
Effect.map(
1173+
serverSettingsService.getSettings,
1174+
(settings) => resolveProjectSettings(settings, projectId).settings.responseStreamingMode,
1175+
);
1176+
1177+
// `mode` is "turn" or "paragraph"; token mode never buffers.
1178+
const appendBufferedAssistantText = (
1179+
messageId: MessageId,
1180+
delta: string,
1181+
mode: Exclude<ResponseStreamingMode, "token">,
1182+
atMillis: number,
1183+
) =>
11711184
Cache.getOption(bufferedAssistantTextByMessageId, messageId).pipe(
11721185
Effect.flatMap((existingText) =>
11731186
Effect.gen(function* () {
@@ -1176,9 +1189,13 @@ const make = Effect.gen(function* () {
11761189
onSome: (text) => `${text}${delta}`,
11771190
});
11781191

1179-
// Deliver finished paragraphs and closed code blocks early so the
1180-
// user sees progress without token-by-token repaints.
1181-
const { ready, rest } = splitBufferedAssistantText(nextText);
1192+
// Paragraph mode delivers finished paragraphs and closed code blocks
1193+
// early so the user sees progress without token-by-token repaints.
1194+
// Turn mode holds everything until the turn finishes or pauses.
1195+
const { ready, rest } =
1196+
mode === "paragraph"
1197+
? splitBufferedAssistantText(nextText)
1198+
: { ready: "", rest: nextText };
11821199
const lastDeliveredAt = Option.getOrUndefined(
11831200
yield* Cache.getOption(lastAssistantDeliveryAtByMessageId, messageId),
11841201
);
@@ -1757,19 +1774,14 @@ const make = Effect.gen(function* () {
17571774
yield* rememberAssistantMessageId(thread.id, turnId, assistantMessageId);
17581775
}
17591776

1760-
const assistantDeliveryMode: AssistantDeliveryMode = yield* Effect.map(
1761-
serverSettingsService.getSettings,
1762-
(settings) =>
1763-
resolveProjectSettings(settings, thread.projectId).settings.enableLegacyTokenStreaming
1764-
? "streaming"
1765-
: "buffered",
1766-
);
1767-
if (assistantDeliveryMode === "buffered") {
1777+
const streamingMode = yield* resolveResponseStreamingMode(thread.projectId);
1778+
if (streamingMode !== "token") {
17681779
// Pace on the server clock. OpenCode stamps every delta of a part
17691780
// with the part's start time, so the event time cannot measure gaps.
17701781
const spillChunk = yield* appendBufferedAssistantText(
17711782
assistantMessageId,
17721783
assistantDelta,
1784+
streamingMode,
17731785
yield* Clock.currentTimeMillis,
17741786
);
17751787
if (spillChunk.length > 0) {
@@ -1807,15 +1819,9 @@ const make = Effect.gen(function* () {
18071819
turnId: pauseForUserTurnId,
18081820
streamingOnly: true,
18091821
});
1810-
const assistantDeliveryMode: AssistantDeliveryMode = yield* Effect.map(
1811-
serverSettingsService.getSettings,
1812-
(settings) =>
1813-
resolveProjectSettings(settings, thread.projectId).settings.enableLegacyTokenStreaming
1814-
? "streaming"
1815-
: "buffered",
1816-
);
1822+
const streamingMode = yield* resolveResponseStreamingMode(thread.projectId);
18171823
const flushedMessageIds =
1818-
assistantDeliveryMode === "buffered"
1824+
streamingMode !== "token"
18191825
? yield* flushBufferedAssistantMessagesForTurn({
18201826
event,
18211827
threadId: thread.id,

0 commit comments

Comments
 (0)