Skip to content

Commit 22aa4a4

Browse files
fix(server): a provider switch mid-turn no longer leaves tool calls running
Switching a thread to another provider while a turn is running releases the old provider session, which fails the run's event stream before the provider can report the turn's end. The fallback that then fails the run closed open subagents but not the run's open tool calls, so a call such as the agent's own t3_thread_configure stayed "running" and the thread kept showing "Waiting on background tasks" until the server restarted. The run now tracks its open command and dynamic tool calls (and their nodes) and ends them as interrupted when its provider stream is lost.
1 parent 0b6ec43 commit 22aa4a4

3 files changed

Lines changed: 373 additions & 5 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3456,6 +3456,57 @@ it.effect("refreshes pull requests after a provider stream exits with an error",
34563456
}),
34573457
);
34583458

3459+
it.effect("closes the run's open tool calls when its provider stream is lost", () =>
3460+
Effect.gen(function* () {
3461+
const key = "lost-stream-open-tools";
3462+
const toolItemId = TurnItemId.make(`turn-item:${key}`);
3463+
const finishedItemId = TurnItemId.make(`turn-item:${key}:finished`);
3464+
const commandItemId = TurnItemId.make(`turn-item:${key}:command`);
3465+
const imageData = "iVBORw0KGgo".repeat(16);
3466+
const { written, observed } = yield* captureRootRunTermination({
3467+
key,
3468+
shouldFinalizeRun: () => Effect.succeed(true),
3469+
events: (ids) =>
3470+
Stream.concat(
3471+
Stream.fromIterable([
3472+
backgroundTurnItemEvent(ids, "dynamic_tool", "running", 1, finishedItemId),
3473+
backgroundTurnItemEvent(ids, "dynamic_tool", "completed", 2, finishedItemId),
3474+
toolCallNodeEvent(ids, "running"),
3475+
withToolOutput(
3476+
withToolCallNode(ids, backgroundTurnItemEvent(ids, "dynamic_tool", "running", 3)),
3477+
{ source: { type: "base64", media_type: "image/png", data: imageData } },
3478+
),
3479+
backgroundTurnItemEvent(ids, "command_execution", "running", 4, commandItemId),
3480+
]),
3481+
// A provider switch releases the session, which fails its event stream.
3482+
Stream.fail(
3483+
new ProviderAdapterEventStreamError({
3484+
driver,
3485+
providerSessionId: ProviderSessionId.make(`session:${key}`),
3486+
cause: "Provider session released: provider_switch.",
3487+
}),
3488+
),
3489+
),
3490+
});
3491+
assert.deepEqual(observed, [
3492+
`node:${toolItemId}:node:interrupted`,
3493+
"run:failed",
3494+
"pull-requests-refreshed",
3495+
]);
3496+
assert.deepEqual(
3497+
written.map((item) => [item.type, item.type === "error" ? null : item.id, item.status]),
3498+
[
3499+
["dynamic_tool", toolItemId, "interrupted"],
3500+
["command_execution", commandItemId, "interrupted"],
3501+
["error", null, "failed"],
3502+
],
3503+
);
3504+
// Closed as the ingestor stored it: without the unserved image bytes.
3505+
const closedTool = written.find((item) => item.id === toolItemId);
3506+
assert.isFalse(JSON.stringify(closedTool).includes(imageData));
3507+
}),
3508+
);
3509+
34593510
it.effect("refreshes pull requests only once when startup failure closes its event stream", () =>
34603511
Effect.gen(function* () {
34613512
const ingestionStarted = yield* Deferred.make<void>();
@@ -3533,6 +3584,12 @@ function captureRootRunTermination(input: {
35333584
if (event.type === "turn-item.updated") {
35343585
yield* captureTurnItem(event.payload);
35353586
}
3587+
if (event.type === "node.updated" && event.payload.kind === "tool_call") {
3588+
yield* Ref.update(observed, (current) => [
3589+
...current,
3590+
`node:${event.payload.id}:${event.payload.status}`,
3591+
]);
3592+
}
35363593
if (event.type === "run.updated") {
35373594
yield* Ref.update(observed, (current) => [...current, `run:${event.payload.status}`]);
35383595
}
@@ -3773,6 +3830,40 @@ function backgroundTurnItemEvent(
37733830
} as ProviderAdapterV2Event;
37743831
}
37753832

3833+
function toolCallNodeEvent(
3834+
ids: BackgroundScenarioIds,
3835+
status: "running" | "completed",
3836+
): ProviderAdapterV2Event {
3837+
return {
3838+
type: "node.updated",
3839+
driver,
3840+
node: {
3841+
id: NodeId.make(`${ids.itemId}:node`),
3842+
threadId: ids.threadId,
3843+
runId: ids.runId,
3844+
kind: "tool_call",
3845+
status,
3846+
},
3847+
} as ProviderAdapterV2Event;
3848+
}
3849+
3850+
function withToolCallNode(
3851+
ids: BackgroundScenarioIds,
3852+
event: ProviderAdapterV2Event,
3853+
): ProviderAdapterV2Event {
3854+
if (event.type !== "turn_item.updated") {
3855+
return event;
3856+
}
3857+
return { ...event, turnItem: { ...event.turnItem, nodeId: NodeId.make(`${ids.itemId}:node`) } };
3858+
}
3859+
3860+
function withToolOutput(event: ProviderAdapterV2Event, output: unknown): ProviderAdapterV2Event {
3861+
if (event.type !== "turn_item.updated") {
3862+
return event;
3863+
}
3864+
return { ...event, turnItem: { ...event.turnItem, output } } as ProviderAdapterV2Event;
3865+
}
3866+
37763867
function backgroundTurnItemEventForRun(
37773868
ids: BackgroundScenarioIds,
37783869
runId: RunId,

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 119 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
5252
import type { ProjectionStoreV2Error } from "./ProjectionStore.ts";
5353
import { makeProviderFailure, makeProviderFailureTurnItem } from "./ProviderFailure.ts";
5454
import * as RunFinalizationService from "./RunFinalizationService.ts";
55+
import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
5556

5657
export interface ProviderEventRoutingState {
5758
readonly ownedThreadIds: ReadonlySet<ThreadId>;
@@ -318,6 +319,71 @@ export function cascadeTerminalizeRunOwnedSubagents(input: {
318319
});
319320
}
320321

322+
/**
323+
* A root run's tool calls and their nodes that have not ended yet. Items are
324+
* kept as the ingestor persists them, without unserved image bytes.
325+
*/
326+
type OpenRunToolCalls = {
327+
readonly turnItems: ReadonlyMap<TurnItemId, OrchestrationV2TurnItem>;
328+
readonly nodes: ReadonlyMap<NodeId, OrchestrationV2ExecutionNode>;
329+
};
330+
331+
function emptyOpenRunToolCalls(): OpenRunToolCalls {
332+
return { turnItems: new Map(), nodes: new Map() };
333+
}
334+
335+
/**
336+
* Ends the tool calls a run left open when its provider event stream was lost,
337+
* such as when a provider switch released the session mid-turn. No process is
338+
* left to report how they ended, so they would otherwise stay running. They
339+
* were cut short, not failed: a call such as t3_thread_configure may have done
340+
* its work before the session went away.
341+
*/
342+
function interruptOpenRunToolCalls(input: {
343+
readonly run: OrchestrationV2Run;
344+
readonly open: OpenRunToolCalls;
345+
readonly completedAt: DateTime.Utc;
346+
readonly allocateEventId: () => Effect.Effect<EventId, IdAllocator.IdAllocatorV2AllocationError>;
347+
}): Effect.Effect<
348+
ReadonlyArray<OrchestrationV2DomainEvent>,
349+
IdAllocator.IdAllocatorV2AllocationError
350+
> {
351+
return Effect.gen(function* () {
352+
const events: Array<OrchestrationV2DomainEvent> = [];
353+
for (const turnItem of input.open.turnItems.values()) {
354+
const node = turnItem.nodeId === null ? undefined : input.open.nodes.get(turnItem.nodeId);
355+
if (node !== undefined) {
356+
events.push({
357+
id: yield* input.allocateEventId(),
358+
type: "node.updated",
359+
threadId: node.threadId,
360+
runId: input.run.id,
361+
nodeId: node.id,
362+
providerInstanceId: input.run.providerInstanceId,
363+
occurredAt: input.completedAt,
364+
payload: { ...node, status: "interrupted", completedAt: input.completedAt },
365+
});
366+
}
367+
events.push({
368+
id: yield* input.allocateEventId(),
369+
type: "turn-item.updated",
370+
threadId: turnItem.threadId,
371+
runId: input.run.id,
372+
...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }),
373+
providerInstanceId: input.run.providerInstanceId,
374+
occurredAt: input.completedAt,
375+
payload: {
376+
...turnItem,
377+
status: "interrupted",
378+
completedAt: input.completedAt,
379+
updatedAt: input.completedAt,
380+
},
381+
});
382+
}
383+
return events;
384+
});
385+
}
386+
321387
export function finalProviderThreadStatus(
322388
disposition: ProviderTerminalEvent["threadDisposition"],
323389
): OrchestrationV2ProviderThread["status"] {
@@ -566,6 +632,8 @@ export const layer: Layer.Layer<
566632
readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
567633
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
568634
readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
635+
/** Set only when the provider stream was lost before the turn ended. */
636+
readonly abandonedToolCalls?: OpenRunToolCalls;
569637
readonly terminal: ProviderTerminalEvent;
570638
readonly failureItemPersisted: boolean;
571639
readonly refreshAfterTurn: Effect.Effect<void>;
@@ -636,6 +704,15 @@ export const layer: Layer.Layer<
636704
allocateEventId,
637705
})
638706
: [];
707+
const abandonedToolCallEvents =
708+
input.abandonedToolCalls === undefined
709+
? []
710+
: yield* interruptOpenRunToolCalls({
711+
run: input.run,
712+
open: input.abandonedToolCalls,
713+
completedAt,
714+
allocateEventId,
715+
});
639716
const persistedStatus =
640717
input.terminal.status === "completed" ? "waiting" : input.terminal.status;
641718
// Completion cohorts are advanced by Orchestrator while a provider
@@ -690,6 +767,7 @@ export const layer: Layer.Layer<
690767
// Terminalize open run-owned subagent rows before the root run
691768
// settles so projections never keep a forever-running subagent card.
692769
...cascadedSubagentEvents,
770+
...abandonedToolCallEvents,
693771
...(finalizedAttempt === null
694772
? []
695773
: [
@@ -960,6 +1038,7 @@ export const layer: Layer.Layer<
9601038
ReadonlySet<OrchestrationV2TurnItem["id"]>
9611039
>(new Set(inheritedBackgroundTurnItemsById.keys()));
9621040
const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1041+
const openToolCalls = yield* Ref.make(emptyOpenRunToolCalls());
9631042
const finalizeRootRun = (terminal: ProviderTerminalEvent) =>
9641043
Effect.gen(function* () {
9651044
if (yield* Ref.get(rootRunFinalized)) {
@@ -1055,6 +1134,23 @@ export const layer: Layer.Layer<
10551134
});
10561135
}
10571136
}
1137+
if (
1138+
event.type === "node.updated" &&
1139+
event.node.kind === "tool_call" &&
1140+
event.node.threadId === input.run.threadId &&
1141+
event.node.runId === input.run.id
1142+
) {
1143+
const node = event.node;
1144+
yield* Ref.update(openToolCalls, (current) => {
1145+
const nodes = new Map(current.nodes);
1146+
if (isOpenExecutionNodeStatus(node.status)) {
1147+
nodes.set(node.id, node);
1148+
} else {
1149+
nodes.delete(node.id);
1150+
}
1151+
return { ...current, nodes };
1152+
});
1153+
}
10581154
if (event.type === "node.updated") {
10591155
const belongsToRootSubagent =
10601156
event.node.kind === "subagent" && event.node.runId === input.run.id;
@@ -1110,6 +1206,23 @@ export const layer: Layer.Layer<
11101206
return { ...current, childTurnItems };
11111207
});
11121208
}
1209+
if (
1210+
belongsToRootRun &&
1211+
event.turnItem.threadId === input.run.threadId &&
1212+
(event.turnItem.type === "command_execution" ||
1213+
event.turnItem.type === "dynamic_tool")
1214+
) {
1215+
const toolItem = event.turnItem;
1216+
yield* Ref.update(openToolCalls, (current) => {
1217+
const turnItems = new Map(current.turnItems);
1218+
if (isSettledTurnItemStatus(toolItem.status)) {
1219+
turnItems.delete(toolItem.id);
1220+
} else {
1221+
turnItems.set(toolItem.id, stripUnservedToolOutputImageBytes(toolItem));
1222+
}
1223+
return { ...current, turnItems };
1224+
});
1225+
}
11131226
if (belongsToRootRun && event.turnItem.type === "subagent") {
11141227
const subagentItem = event.turnItem;
11151228
yield* Ref.update(openRunOwnedSubagents, (current) => {
@@ -1293,8 +1406,11 @@ export const layer: Layer.Layer<
12931406
Effect.flatMap((providerThread) =>
12941407
Ref.get(latestTurnItemOrdinal).pipe(
12951408
Effect.flatMap((latestItemOrdinal) =>
1296-
Ref.get(openRunOwnedSubagents).pipe(
1297-
Effect.flatMap((openSubagents) =>
1409+
Effect.all([
1410+
Ref.get(openRunOwnedSubagents),
1411+
Ref.get(openToolCalls),
1412+
]).pipe(
1413+
Effect.flatMap(([openSubagents, abandonedToolCalls]) =>
12981414
writeFinalRunEvents({
12991415
run: input.run,
13001416
rootNode: input.rootNode,
@@ -1308,6 +1424,7 @@ export const layer: Layer.Layer<
13081424
expectedStatus: "running",
13091425
},
13101426
openRunOwnedSubagents: openSubagents,
1427+
abandonedToolCalls,
13111428
terminal: makeFailedTerminalEvent(
13121429
makeProviderFailure({
13131430
cause: Cause.squash(cause),

0 commit comments

Comments
 (0)