Skip to content

Commit 0e40614

Browse files
StiensWoutclaude
andauthored
perf(acp): coalesce streamed text snapshots (#19626)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent e9bd77e commit 0e40614

2 files changed

Lines changed: 201 additions & 7 deletions

File tree

‎apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1944,6 +1944,144 @@ describe("AcpAdapterV2", () => {
19441944
}).pipe(Effect.provide(layerTest), Effect.scoped),
19451945
);
19461946

1947+
it.effect("coalesces streamed ACP text into bounded snapshots without delaying its start", () =>
1948+
Effect.gen(function* () {
1949+
const childProcessSpawner = yield* ChildProcessSpawner.ChildProcessSpawner;
1950+
const path = yield* Path.Path;
1951+
const selfInvocation = yield* resolveSelfInvocation();
1952+
const mockAgentPath = yield* path.fromFileUrl(
1953+
new URL("../../../scripts/acp-mock-agent.ts", import.meta.url),
1954+
);
1955+
type RuntimeService = AcpSessionRuntime.AcpSessionRuntime["Service"];
1956+
let sessionUpdateHandler: Parameters<RuntimeService["handleSessionUpdate"]>[0] | undefined;
1957+
const protocolEvents = yield* Queue.unbounded<EffectAcpProtocol.AcpProtocolLogEvent>();
1958+
const instanceId = ProviderInstanceId.make("acp-test-text-coalescing");
1959+
const adapter = yield* makeAcpAdapterV2({
1960+
instanceId,
1961+
flavor: {
1962+
driver: ACP_TEST_DRIVER,
1963+
capabilities: AcpProviderCapabilitiesV2,
1964+
makeRuntime: makeMockRuntime({
1965+
childProcessSpawner,
1966+
mockAgentPath,
1967+
environment: { T3_ACP_HANG_PROMPT_FOREVER: "1" },
1968+
protocolEvents,
1969+
wrapRuntime: (runtime) => ({
1970+
...runtime,
1971+
handleSessionUpdate: (handler) =>
1972+
Effect.sync(() => {
1973+
sessionUpdateHandler = handler;
1974+
}).pipe(Effect.andThen(runtime.handleSessionUpdate(handler))),
1975+
}),
1976+
}),
1977+
},
1978+
selfInvocation,
1979+
});
1980+
const threadId = ThreadId.make("thread-acp-text-coalescing");
1981+
const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({
1982+
runtimeMode: "full-access",
1983+
interactionMode: "default",
1984+
cwd: process.cwd(),
1985+
});
1986+
const modelSelection = { instanceId, model: "default" } as const;
1987+
const runtime = yield* adapter.openSession({
1988+
threadId,
1989+
providerSessionId: ProviderSessionId.make("provider-session-acp-text-coalescing"),
1990+
modelSelection,
1991+
runtimePolicy,
1992+
});
1993+
const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
1994+
yield* runtime.events.pipe(
1995+
Stream.runForEach((event) => Queue.offer(events, event)),
1996+
Effect.forkScoped,
1997+
);
1998+
const providerThread = yield* runtime.ensureThread({
1999+
threadId,
2000+
modelSelection,
2001+
runtimePolicy,
2002+
});
2003+
const now = yield* DateTime.now;
2004+
yield* runtime.startTurn(
2005+
makeTurnInput({ threadId, providerThread, instanceId, runtimePolicy, now }),
2006+
);
2007+
yield* Stream.fromQueue(protocolEvents).pipe(
2008+
Stream.filter(
2009+
(event) =>
2010+
event.direction === "outgoing" && rawProtocolMethod(event) === "session/prompt",
2011+
),
2012+
Stream.runHead,
2013+
);
2014+
assert.isDefined(sessionUpdateHandler);
2015+
const sendUpdate = (update: EffectAcpSchema.SessionNotification["update"]) =>
2016+
sessionUpdateHandler!({ sessionId: "mock-session-1", update });
2017+
const chunk = "tok0 word ";
2018+
2019+
for (let index = 0; index < 1_000; index += 1) {
2020+
yield* sendUpdate({
2021+
sessionUpdate: "agent_message_chunk",
2022+
content: { type: "text", text: chunk },
2023+
});
2024+
}
2025+
yield* TestClock.adjust("50 millis");
2026+
yield* sendUpdate({
2027+
sessionUpdate: "agent_message_chunk",
2028+
content: { type: "text", text: "end" },
2029+
});
2030+
// Another item closes the assistant segment while its next flush is pending.
2031+
yield* sendUpdate({
2032+
sessionUpdate: "agent_thought_chunk",
2033+
content: { type: "text", text: "next" },
2034+
});
2035+
yield* TestClock.adjust("50 millis");
2036+
yield* sendUpdate({
2037+
sessionUpdate: "plan_update",
2038+
plan: {
2039+
type: "items",
2040+
planId: "plan-sentinel",
2041+
entries: [{ content: "Continue", priority: "medium", status: "in_progress" }],
2042+
},
2043+
});
2044+
const streamed: Array<ProviderAdapter.ProviderAdapterV2Event> = [];
2045+
while (streamed.at(-1)?.type !== "plan.updated") {
2046+
streamed.push(yield* Queue.take(events));
2047+
}
2048+
2049+
const reply = chunk.repeat(1_000);
2050+
assert.deepEqual(
2051+
streamed.flatMap((event) =>
2052+
event.type === "turn_item.updated" && event.turnItem.type === "assistant_message"
2053+
? [[event.turnItem.status, event.turnItem.text]]
2054+
: [],
2055+
),
2056+
[
2057+
["running", chunk],
2058+
["running", reply],
2059+
["completed", `${reply}end`],
2060+
],
2061+
);
2062+
assert.deepEqual(
2063+
streamed.flatMap((event) =>
2064+
event.type === "message.updated" && event.message.role === "assistant"
2065+
? [[event.message.streaming, event.message.text]]
2066+
: [],
2067+
),
2068+
[
2069+
[true, chunk],
2070+
[true, reply],
2071+
[false, `${reply}end`],
2072+
],
2073+
);
2074+
assert.deepEqual(
2075+
streamed.flatMap((event) =>
2076+
event.type === "node.updated" && event.node.kind === "assistant_message"
2077+
? [event.node.status]
2078+
: [],
2079+
),
2080+
["running", "completed"],
2081+
);
2082+
}).pipe(Effect.provide(layerTest), Effect.scoped),
2083+
);
2084+
19472085
it.live(
19482086
"loads the persisted ACP session during startup without creating a throwaway session",
19492087
() =>

‎packages/provider-acp/src/server/adapter.ts‎

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,15 @@ export const ACP_PROTOCOL = "acp.ndjson-jsonrpc" as const;
128128
*/
129129
const ACP_DEFERRED_FINALIZE_DEBOUNCE: Duration.Input = "3000 millis";
130130

131+
/**
132+
* Coalescing window for streamed text. A text segment's first chunk and its
133+
* close emit at once; a later chunk schedules one full-text snapshot this long
134+
* after it, and the chunks inside the window share it instead of each emitting
135+
* the whole reply, which grew with the square of the reply length. The flush
136+
* is scheduled, not a hard deadline, so it can land later under host load.
137+
*/
138+
const ACP_TEXT_FLUSH_INTERVAL: Duration.Input = "50 millis";
139+
131140
export interface AcpAdapterV2RuntimeInput {
132141
readonly cwd: string;
133142
readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
@@ -957,6 +966,10 @@ interface ActiveTextSegment {
957966
readonly startedAt: DateTime.Utc;
958967
sourceMessageId: string | null;
959968
text: string;
969+
// The running node and first text have been emitted.
970+
emitted: boolean;
971+
// Text changed since the last emission; a delayed flush is scheduled.
972+
dirty: boolean;
960973
}
961974

962975
interface ActiveTextStream {
@@ -2168,14 +2181,32 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
21682181
? context.assistant
21692182
: context.reasoning;
21702183

2184+
const textEmissionPermit = yield* Semaphore.make(1);
2185+
/**
2186+
* Emits the stream's current segment. A `flushOf` call is a delayed
2187+
* flush: it emits only while that segment is still current and has
2188+
* unsent text. The permit keeps a flush from landing after the
2189+
* segment's completed snapshot.
2190+
*/
21712191
const emitTextSegment = Effect.fnUntraced(function* (
21722192
context: ActiveAcpTurn,
21732193
kind: "user" | "assistant" | "reasoning",
21742194
completed: boolean,
2195+
flushOf?: ActiveTextSegment,
21752196
) {
21762197
const stream = textStreamFor(context, kind);
21772198
const segment = stream.current;
21782199
if (segment === null) return;
2200+
if (flushOf !== undefined && (segment !== flushOf || !segment.dirty)) return;
2201+
// The node carries no text, so it changes only when the segment
2202+
// opens and when it completes.
2203+
const emitsNode = completed || !segment.emitted;
2204+
segment.emitted = true;
2205+
segment.dirty = false;
2206+
// Read the text in the same step that clears `dirty`. Chunks keep
2207+
// appending while this emission yields; each one marks the segment
2208+
// dirty again and reaches the next flush or the close.
2209+
const text = segment.text;
21792210
const now = yield* DateTime.now;
21802211
const ordinal = yield* resolveItemOrdinal(context, segment.nativeItemId);
21812212
const nodeId = providerNodeId(segment.nativeItemId);
@@ -2185,7 +2216,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
21852216
nativeId: segment.nativeItemId,
21862217
strength: "weak" as const,
21872218
};
2188-
if (kind !== "user") {
2219+
if (kind !== "user" && emitsNode) {
21892220
yield* emitProviderEvent({
21902221
type: "node.updated",
21912222
driver,
@@ -2219,7 +2250,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
22192250
runId: context.input.runId,
22202251
nodeId: messageNodeId,
22212252
role: kind,
2222-
text: segment.text,
2253+
text,
22232254
attachments: [],
22242255
streaming: !completed,
22252256
createdAt: segment.startedAt,
@@ -2251,7 +2282,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
22512282
type: "user_message",
22522283
messageId,
22532284
inputIntent: "turn_start",
2254-
text: segment.text,
2285+
text,
22552286
attachments: [],
22562287
},
22572288
}
@@ -2275,7 +2306,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
22752306
updatedAt: now,
22762307
type: "assistant_message",
22772308
messageId,
2278-
text: segment.text,
2309+
text,
22792310
streaming: !completed,
22802311
},
22812312
},
@@ -2302,10 +2333,31 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
23022333
completedAt: completed ? now : null,
23032334
updatedAt: now,
23042335
type: "reasoning",
2305-
text: segment.text,
2336+
text,
23062337
streaming: !completed,
23072338
},
23082339
});
2340+
}, textEmissionPermit.withPermit);
2341+
2342+
/**
2343+
* Publishes a text change. A segment's first text emits at once so
2344+
* live output starts immediately; later changes coalesce into one
2345+
* snapshot per ACP_TEXT_FLUSH_INTERVAL. Closing the segment emits its
2346+
* final text and makes a pending flush a no-op.
2347+
*/
2348+
const publishTextSegment = Effect.fnUntraced(function* (
2349+
context: ActiveAcpTurn,
2350+
kind: "user" | "assistant" | "reasoning",
2351+
) {
2352+
const segment = textStreamFor(context, kind).current;
2353+
if (segment === null) return;
2354+
if (!segment.emitted) return yield* emitTextSegment(context, kind, false);
2355+
if (segment.dirty) return;
2356+
segment.dirty = true;
2357+
yield* emitTextSegment(context, kind, false, segment).pipe(
2358+
Effect.delay(ACP_TEXT_FLUSH_INTERVAL),
2359+
Effect.forkIn(sessionScope),
2360+
);
23092361
});
23102362

23112363
const closeTextStream = Effect.fnUntraced(function* (
@@ -2354,6 +2406,8 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
23542406
startedAt: now,
23552407
sourceMessageId,
23562408
text: "",
2409+
emitted: false,
2410+
dirty: false,
23572411
};
23582412
stream.nextSegment += 1;
23592413
} else if (stream.current.sourceMessageId === null && sourceMessageId !== null) {
@@ -2362,7 +2416,7 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
23622416
stream.current.sourceMessageId = sourceMessageId;
23632417
}
23642418
stream.current.text += text;
2365-
yield* emitTextSegment(context, kind, false);
2419+
yield* publishTextSegment(context, kind);
23662420
});
23672421

23682422
const replaceText = Effect.fnUntraced(function* (
@@ -2383,12 +2437,14 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* (
23832437
startedAt: now,
23842438
sourceMessageId: messageId,
23852439
text,
2440+
emitted: false,
2441+
dirty: false,
23862442
};
23872443
stream.nextSegment += 1;
23882444
} else {
23892445
stream.current.text = text;
23902446
}
2391-
yield* emitTextSegment(context, kind, false);
2447+
yield* publishTextSegment(context, kind);
23922448
});
23932449

23942450
const emitSubagentAssistant = Effect.fnUntraced(function* (

0 commit comments

Comments
 (0)