Skip to content

Commit c784f2c

Browse files
committed
fix(orchestration-v2): recover runs with terminal provider turns
1 parent 211d024 commit c784f2c

2 files changed

Lines changed: 62 additions & 18 deletions

File tree

‎apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,12 @@ const stopEarlierBackgroundWork = ({
5151
readonly failedStart?: boolean;
5252
readonly stopWithQueue?: "thread.stop" | "run.interrupt";
5353
readonly olderStart?: boolean;
54-
readonly stalledRun?: "missing-session" | "returned-interrupt" | "superseded-attempt";
54+
readonly stalledRun?:
55+
| "missing-session"
56+
| "missing-session-terminal"
57+
| "returned-interrupt"
58+
| "returned-interrupt-terminal"
59+
| "superseded-attempt";
5560
}) =>
5661
Effect.scoped(
5762
Effect.gen(function* () {
@@ -218,7 +223,7 @@ const stopEarlierBackgroundWork = ({
218223
const run = before.runs[0]!;
219224
const node = before.nodes.find((candidate) => candidate.id === run.rootNodeId)!;
220225
const attempt = before.attempts[0]!;
221-
if (stalledRun === "missing-session") {
226+
if (stalledRun === "missing-session" || stalledRun === "missing-session-terminal") {
222227
const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
223228
const failed = yield* watch(
224229
(event) => event.type === "run.updated" && event.payload.status === "failed",
@@ -258,6 +263,31 @@ const stopEarlierBackgroundWork = ({
258263
const messageId = MessageId.make("partial-output");
259264
const item = before.turnItems.find((candidate) => candidate.id === devServerId)!;
260265
assert.ok(item.type === "command_execution");
266+
const terminalProviderTurn = stalledRun.endsWith("-terminal");
267+
if (terminalProviderTurn) {
268+
yield* sink.write({
269+
events: [
270+
{
271+
id: EventId.make("terminal-provider-turn"),
272+
type: "provider-turn.updated",
273+
threadId,
274+
occurredAt: now,
275+
payload: { ...codexTurn, status: "completed", completedAt: now },
276+
},
277+
...(stalledRun === "missing-session-terminal"
278+
? [
279+
{
280+
id: EventId.make("completed-dev-server"),
281+
type: "turn-item.updated" as const,
282+
threadId,
283+
occurredAt: now,
284+
payload: { ...item, status: "completed" as const, completedAt: now },
285+
},
286+
]
287+
: []),
288+
],
289+
});
290+
}
261291
yield* sink.write({
262292
events: [
263293
{
@@ -333,7 +363,10 @@ const stopEarlierBackgroundWork = ({
333363
const interrupted = stalledRun !== "superseded-attempt";
334364
assert.equal(after.runs[0]?.status, interrupted ? "interrupted" : "running");
335365
assert.equal(after.attempts[0]?.status, interrupted ? "interrupted" : "running");
336-
assert.equal(after.providerTurns[0]?.status, interrupted ? "interrupted" : "running");
366+
assert.equal(
367+
after.providerTurns[0]?.status,
368+
terminalProviderTurn ? "completed" : interrupted ? "interrupted" : "running",
369+
);
337370
assert.equal(
338371
after.nodes.find((candidate) => candidate.id === node.id)?.status,
339372
interrupted ? "interrupted" : "running",
@@ -343,7 +376,11 @@ const stopEarlierBackgroundWork = ({
343376
assert.equal(output.text, "Partial output");
344377
assert.equal(
345378
after.turnItems.find((candidate) => candidate.id === devServerId)?.status,
346-
interrupted ? "interrupted" : "running",
379+
stalledRun === "missing-session-terminal"
380+
? "completed"
381+
: interrupted
382+
? "interrupted"
383+
: "running",
347384
);
348385
assert.equal(
349386
after.turnItems.find((candidate) => candidate.type === "assistant_message")?.status,
@@ -746,7 +783,13 @@ it.effect.each(["thread.stop", "run.interrupt"] as const)(
746783
(stopType) => stopEarlierBackgroundWork({ stopWithQueue: stopType, olderStart: true }),
747784
);
748785

749-
it.effect.each(["missing-session", "returned-interrupt", "superseded-attempt"] as const)(
786+
it.effect.each([
787+
"missing-session",
788+
"missing-session-terminal",
789+
"returned-interrupt",
790+
"returned-interrupt-terminal",
791+
"superseded-attempt",
792+
] as const)(
750793
"Stop recovers a stalled run after %s without changing a newer attempt",
751794
(stalledRun) => stopEarlierBackgroundWork({ stalledRun }),
752795
);

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8167,8 +8167,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
81678167
rootNode === undefined ||
81688168
run.activeAttemptId !== attempt.id ||
81698169
run.status !== "running" ||
8170-
attempt.status !== "running" ||
8171-
input.providerTurn.status !== "running"
8170+
attempt.status !== "running"
81728171
)
81738172
return;
81748173
const emitEvent = emit(input.events, input.command);
@@ -8179,15 +8178,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
81798178
providerInstanceId: run.providerInstanceId,
81808179
occurredAt: input.now,
81818180
};
8182-
yield* emitEvent({
8183-
...base,
8184-
type: "provider-turn.updated",
8185-
payload: {
8186-
...input.providerTurn,
8187-
status: "interrupted",
8188-
completedAt: input.now,
8189-
},
8190-
});
8181+
if (input.providerTurn.status === "running") {
8182+
yield* emitEvent({
8183+
...base,
8184+
type: "provider-turn.updated",
8185+
payload: {
8186+
...input.providerTurn,
8187+
status: "interrupted",
8188+
completedAt: input.now,
8189+
},
8190+
});
8191+
}
81918192
yield* emitEvent({
81928193
...base,
81938194
type: "run-attempt.updated",
@@ -8521,7 +8522,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
85218522
)
85228523
return;
85238524
const now = yield* DateTime.now;
8524-
if (stopped.providerTurn?.status === "running" && stoppedRun.status === "running") {
8525+
if (stopped.providerTurn !== undefined && stoppedRun.status === "running") {
85258526
const output = yield* projectionStore
85268527
.getThreadRecords(command.threadId, ["messages"], {
85278528
messageRunIds: [stoppedRun.id],
@@ -8613,7 +8614,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
86138614
projection.providerTurns.findLast(
86148615
(candidate) =>
86158616
candidate.runAttemptId === run?.activeAttemptId &&
8616-
(candidate.status === "running" || hasBackgroundWork),
8617+
(run?.status === "running" || candidate.status === "running" || hasBackgroundWork),
86178618
) ??
86188619
(hasBackgroundWork
86198620
? projection.providerTurns.findLast(

0 commit comments

Comments
 (0)