Skip to content

Commit 573d81f

Browse files
committed
fix(server): withdraw failed runs' child approvals
1 parent 7e71853 commit 573d81f

2 files changed

Lines changed: 141 additions & 32 deletions

File tree

‎apps/server/src/orchestration-v2/RunExecutionService.test.ts‎

Lines changed: 90 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2673,12 +2673,18 @@ it.effect("does not refresh pull requests for auxiliary or stale provider termin
26732673
}),
26742674
);
26752675

2676-
it.effect.each(["stream", "terminal", "continuing", "completed"] as const)(
2676+
it.effect.each(["stream", "terminal", "continuing", "completed", "transferred"] as const)(
26772677
"withdraws failed-turn approvals only when children stop (%s)",
26782678
(failureMode) =>
26792679
Effect.gen(function* () {
26802680
const now = yield* DateTime.now;
26812681
const requestId = RuntimeRequestId.make("request:stream-error");
2682+
const childRequestId = RuntimeRequestId.make("request:stream-error:child");
2683+
const childTurnId = ProviderTurnId.make("provider-turn:stream-error:child");
2684+
const childProviderThreadId = ProviderThreadId.make("provider-thread:stream-error:child");
2685+
const childThreadId = backgroundScenarioIds(
2686+
"pull-request-refresh:stream-error",
2687+
).childThreadId;
26822688
const observedTurnId = ProviderTurnId.make("provider-turn:stream-error:observed");
26832689
const guarded = yield* Ref.make(false);
26842690
const { written, observed, finalEvents } = yield* captureRootRunTermination({
@@ -2689,15 +2695,16 @@ it.effect.each(["stream", "terminal", "continuing", "completed"] as const)(
26892695
Effect.succeed(
26902696
input.withdrawApprovals &&
26912697
input.event.type === "turn.terminal" &&
2692-
input.event.providerTurnId === observedTurnId
2698+
(input.event.providerTurnId === observedTurnId ||
2699+
(input.event.providerTurnId === childTurnId && input.threadId === childThreadId))
26932700
? [
26942701
{
2695-
id: EventId.make("event:stream-error:withdraw-request"),
2702+
id: EventId.make(`event:stream-error:withdraw-request:${input.threadId}`),
26962703
type: "runtime-request.updated",
26972704
threadId: input.threadId,
26982705
occurredAt: now,
26992706
payload: {
2700-
id: requestId,
2707+
id: input.threadId === childThreadId ? childRequestId : requestId,
27012708
nodeId: NodeId.make("node:stream-error:approval"),
27022709
providerTurnId:
27032710
input.event.type === "turn.terminal" ? input.event.providerTurnId : null,
@@ -2722,21 +2729,78 @@ it.effect.each(["stream", "terminal", "continuing", "completed"] as const)(
27222729
},
27232730
events: (ids) =>
27242731
Stream.concat(
2725-
Stream.make({
2726-
type: "provider_turn.updated",
2727-
driver,
2728-
providerTurn: {
2729-
id: observedTurnId,
2730-
providerThreadId: ids.providerThreadId,
2731-
nodeId: ids.rootNodeId,
2732-
runAttemptId: ids.attemptId,
2733-
nativeTurnRef: null,
2734-
ordinal: 1,
2735-
status: "running",
2736-
startedAt: now,
2737-
completedAt: null,
2732+
Stream.make(
2733+
childThreadCreatedEvent(ids),
2734+
{
2735+
type: "provider_turn.updated",
2736+
threadId: childThreadId,
2737+
driver,
2738+
providerTurn: {
2739+
id: childTurnId,
2740+
providerThreadId: childProviderThreadId,
2741+
nodeId: NodeId.make("node:stream-error:child-root"),
2742+
runAttemptId: null,
2743+
nativeTurnRef: null,
2744+
ordinal: 1,
2745+
status: "running",
2746+
startedAt: now,
2747+
completedAt: null,
2748+
},
27382749
},
2739-
}),
2750+
{
2751+
type: "runtime_request.updated",
2752+
driver,
2753+
threadId: childThreadId,
2754+
runtimeRequest: {
2755+
id: childRequestId,
2756+
nodeId: NodeId.make("node:stream-error:child-approval"),
2757+
providerTurnId: childTurnId,
2758+
nativeRequestRef: null,
2759+
kind: "command",
2760+
status: "pending",
2761+
responseCapability: {
2762+
type: "live",
2763+
providerSessionId: ProviderSessionId.make("session:stream-error"),
2764+
},
2765+
createdAt: now,
2766+
resolvedAt: null,
2767+
},
2768+
},
2769+
{
2770+
type: "provider_turn.updated",
2771+
driver,
2772+
providerTurn: {
2773+
id: observedTurnId,
2774+
providerThreadId: ids.providerThreadId,
2775+
nodeId: ids.rootNodeId,
2776+
runAttemptId: ids.attemptId,
2777+
nativeTurnRef: null,
2778+
ordinal: 1,
2779+
status: "running",
2780+
startedAt: now,
2781+
completedAt: null,
2782+
},
2783+
},
2784+
...(failureMode === "transferred"
2785+
? [
2786+
{
2787+
type: "subagent.updated" as const,
2788+
driver,
2789+
subagent: {
2790+
...makeRunOwnedSubagentFixture({
2791+
ids,
2792+
providerInstanceId: ProviderInstanceId.make("codex"),
2793+
childThreadId,
2794+
driver,
2795+
status: "running",
2796+
}),
2797+
providerThreadId: childProviderThreadId,
2798+
runId: RunId.make("run:replacement"),
2799+
},
2800+
},
2801+
]
2802+
: []),
2803+
),
27402804
failureMode === "stream"
27412805
? Stream.fail(
27422806
new ProviderAdapter.ProviderAdapterEventStreamError({
@@ -2768,10 +2832,17 @@ it.effect.each(["stream", "terminal", "continuing", "completed"] as const)(
27682832
assert.include(error?.failure.message ?? "", "provider event stream closed unexpectedly");
27692833
assert.equal(
27702834
finalEvents.some(
2771-
(event) => event.type === "runtime-request.updated" && event.payload.id === requestId,
2835+
(event) =>
2836+
event.type === "runtime-request.updated" && event.payload.id === childRequestId,
27722837
),
27732838
failureMode === "stream" || failureMode === "terminal",
27742839
);
2840+
assert.equal(
2841+
finalEvents.some(
2842+
(event) => event.type === "runtime-request.updated" && event.payload.id === requestId,
2843+
),
2844+
failureMode === "stream" || failureMode === "terminal" || failureMode === "transferred",
2845+
);
27752846
}),
27762847
);
27772848

‎apps/server/src/orchestration-v2/RunExecutionService.ts‎

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import {
2121
type ProviderThreadId,
2222
type ProviderTurnId,
2323
type RunAttemptId,
24+
type RuntimeRequestId,
2425
type ThreadId,
2526
type TurnItemId,
2627
} from "@t3tools/contracts";
@@ -152,6 +153,11 @@ export function selectInheritedBackgroundTurnItems(input: {
152153

153154
type SubagentTurnItem = Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>;
154155

156+
type PendingRequestTurn = {
157+
readonly threadId: ThreadId;
158+
readonly providerTurnId: ProviderTurnId;
159+
};
160+
155161
type OpenRunOwnedSubagentProjection = {
156162
readonly subagents: ReadonlyMap<NodeId, OrchestrationV2Subagent>;
157163
readonly turnItems: ReadonlyMap<NodeId, SubagentTurnItem>;
@@ -635,6 +641,7 @@ export const layer: Layer.Layer<
635641
readonly terminal: ProviderTerminalEvent;
636642
readonly completedAt?: DateTime.Utc;
637643
readonly failureItemPersisted: boolean;
644+
readonly pendingRequestTurns?: ReadonlyArray<PendingRequestTurn>;
638645
readonly refreshAfterTurn: Effect.Effect<void>;
639646
readonly writeIfRunCurrent?: {
640647
readonly activeAttemptId: RunAttemptId;
@@ -740,20 +747,33 @@ export const layer: Layer.Layer<
740747
`command:effect:checkpoint.capture:${input.run.id}`,
741748
);
742749
// Failed turns without continuing children have no consumer for their pending callbacks.
743-
const cancelledRequestEvents =
744-
input.terminal.status === "failed" && !childTurnsContinue(input.terminal)
745-
? (yield* providerEventIngestor.normalize({
746-
providerSessionId: input.providerSessionId,
747-
providerInstanceId: input.run.providerInstanceId,
748-
threadId: input.run.threadId,
749-
runId: input.run.id,
750-
nodeId: input.rootNode.id,
751-
event: input.terminal,
752-
withdrawApprovals: true,
753-
})).filter(
750+
const cancelledRequestEvents: Array<OrchestrationV2DomainEvent> = [];
751+
if (input.terminal.status === "failed" && !childTurnsContinue(input.terminal)) {
752+
const turns = new Map<ThreadId, Set<ProviderTurnId>>();
753+
for (const turn of [
754+
{ threadId: input.run.threadId, providerTurnId: input.terminal.providerTurnId },
755+
...(input.pendingRequestTurns ?? []),
756+
]) {
757+
const seen = turns.get(turn.threadId) ?? new Set<ProviderTurnId>();
758+
if (seen.has(turn.providerTurnId)) continue;
759+
seen.add(turn.providerTurnId);
760+
turns.set(turn.threadId, seen);
761+
const events = yield* providerEventIngestor.normalize({
762+
providerSessionId: input.providerSessionId,
763+
providerInstanceId: input.run.providerInstanceId,
764+
threadId: turn.threadId,
765+
runId: input.run.id,
766+
nodeId: input.rootNode.id,
767+
event: { ...input.terminal, providerTurnId: turn.providerTurnId },
768+
withdrawApprovals: true,
769+
});
770+
cancelledRequestEvents.push(
771+
...events.filter(
754772
(event) => event.type !== "turn-item.updated" || event.payload.type !== "error",
755-
)
756-
: [];
773+
),
774+
);
775+
}
776+
}
757777
const finalization = {
758778
guardPendingRuntimeRequestCancellations: true,
759779
// Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane.
@@ -1083,6 +1103,13 @@ export const layer: Layer.Layer<
10831103
: { relatedProviderThreadIds: input.relatedProviderThreadIds }),
10841104
}),
10851105
);
1106+
const pendingRequestTurns = new Map<RuntimeRequestId, PendingRequestTurn>();
1107+
const ownedPendingRequestTurns = (routing: ProviderEventRoutingState) =>
1108+
[...pendingRequestTurns.values()].filter(
1109+
(turn) =>
1110+
routing.ownedProviderTurns.has(turn.providerTurnId) ||
1111+
(turn.threadId !== input.run.threadId && routing.ownedThreadIds.has(turn.threadId)),
1112+
);
10861113
const rootTerminalSeen = yield* Ref.make(false);
10871114
const rootRunFinalized = yield* Ref.make(false);
10881115
const providerThreadOwnerLost = yield* Ref.make(false);
@@ -1128,6 +1155,7 @@ export const layer: Layer.Layer<
11281155
? {}
11291156
: { loadRunInterruptReason: input.loadRunInterruptReason }),
11301157
openRunOwnedSubagents: openSubagents,
1158+
pendingRequestTurns: ownedPendingRequestTurns(yield* Ref.get(eventRouting)),
11311159
terminal,
11321160
failureItemPersisted: terminal.status === "failed",
11331161
refreshAfterTurn,
@@ -1145,6 +1173,15 @@ export const layer: Layer.Layer<
11451173
deliverable: boolean,
11461174
) {
11471175
const routing = yield* Ref.get(eventRouting);
1176+
if (event.type === "runtime_request.updated") {
1177+
const request = event.runtimeRequest;
1178+
if (request.status === "pending" && request.providerTurnId !== null) {
1179+
pendingRequestTurns.set(request.id, {
1180+
threadId: event.threadId ?? input.run.threadId,
1181+
providerTurnId: request.providerTurnId,
1182+
});
1183+
} else pendingRequestTurns.delete(request.id);
1184+
}
11481185
if (event.type === "provider_turn.updated") {
11491186
const isRoot =
11501187
event.providerTurn.runAttemptId === input.attempt.id ||
@@ -1633,6 +1670,7 @@ export const layer: Layer.Layer<
16331670
expectedStatus: "running",
16341671
},
16351672
openRunOwnedSubagents: openSubagents,
1673+
pendingRequestTurns: ownedPendingRequestTurns(routing),
16361674
...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
16371675
completedAt,
16381676
terminal: makeFailedTerminalEvent(

0 commit comments

Comments
 (0)