Skip to content

Commit f0c882d

Browse files
committed
fix(orchestration-v2): preserve native interrupt cleanup and partial output
1 parent 2b612b1 commit f0c882d

5 files changed

Lines changed: 208 additions & 32 deletions

File tree

‎apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ it.effect.each(["missing-session", "returned-interrupt", "superseded-attempt"] a
9191
nativeThreadRef: null,
9292
nativeConversationHeadRef: null,
9393
status: "active",
94+
pendingBackgroundTasks: [
95+
{ kind: "background_task", taskId: "stalled-task", description: "Background task" },
96+
],
9497
firstRunOrdinal: 1,
9598
lastRunOrdinal: 1,
9699
handoffIds: [],
@@ -207,6 +210,57 @@ it.effect.each(["missing-session", "returned-interrupt", "superseded-attempt"] a
207210
input: "test",
208211
},
209212
});
213+
const assistantMessageId = MessageId.make(`${scenario}:assistant`);
214+
yield* projections.apply({
215+
id: EventId.make(`${scenario}:message`),
216+
type: "message.updated",
217+
threadId,
218+
runId,
219+
occurredAt: now,
220+
payload: {
221+
id: assistantMessageId,
222+
threadId,
223+
runId,
224+
nodeId,
225+
role: "assistant",
226+
text: "Partial output",
227+
attachments: [],
228+
streaming: true,
229+
createdBy: "agent",
230+
creationSource: "provider",
231+
createdAt: now,
232+
updatedAt: now,
233+
},
234+
});
235+
for (const type of ["assistant_message", "reasoning"] as const) {
236+
yield* projections.apply({
237+
id: EventId.make(`${scenario}:${type}`),
238+
type: "turn-item.updated",
239+
threadId,
240+
runId,
241+
occurredAt: now,
242+
payload: {
243+
id: TurnItemId.make(`${scenario}:${type}`),
244+
threadId,
245+
runId,
246+
nodeId,
247+
providerThreadId,
248+
providerTurnId,
249+
nativeItemRef: null,
250+
parentItemId: null,
251+
ordinal: type === "assistant_message" ? 2 : 3,
252+
status: "running",
253+
title: "Partial output",
254+
startedAt: now,
255+
completedAt: null,
256+
updatedAt: now,
257+
type,
258+
messageId: assistantMessageId,
259+
text: "Partial output",
260+
streaming: true,
261+
},
262+
});
263+
}
210264
yield* projections.apply({
211265
id: EventId.make(`${scenario}:request`),
212266
type: "runtime-request.updated",
@@ -258,6 +312,16 @@ it.effect.each(["missing-session", "returned-interrupt", "superseded-attempt"] a
258312
assert.equal(after.attempts[0]?.status, interrupted ? "interrupted" : "running");
259313
assert.equal(after.providerTurns[0]?.status, interrupted ? "interrupted" : "running");
260314
assert.equal(after.nodes[0]?.status, interrupted ? "interrupted" : "running");
315+
assert.equal(after.providerThreads[0]?.status, interrupted ? "idle" : "active");
316+
assert.lengthOf(after.providerThreads[0]?.pendingBackgroundTasks ?? [], interrupted ? 0 : 1);
317+
assert.equal(after.messages[0]?.streaming, !interrupted);
318+
assert.equal(after.messages[0]?.text, "Partial output");
319+
for (const item of after.turnItems) {
320+
if (item.type !== "assistant_message" && item.type !== "reasoning") continue;
321+
assert.equal(item.status, interrupted ? "interrupted" : "running");
322+
assert.equal(item.streaming, !interrupted);
323+
assert.equal(item.text, "Partial output");
324+
}
261325
assert.equal(
262326
after.turnItems.find((item) => item.type === "command_execution")?.status,
263327
interrupted ? "interrupted" : "running",

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 84 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7937,7 +7937,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
79377937
driver: providerThread.driver,
79387938
providerInstanceId: providerThread.providerInstanceId,
79397939
occurredAt: input.now,
7940-
payload: { ...providerThread, pendingBackgroundTasks: [], updatedAt: input.now },
7940+
payload: {
7941+
...providerThread,
7942+
status:
7943+
providerThread.status === "active" &&
7944+
providerThread.lastRunOrdinal !== null &&
7945+
providerThread.lastRunOrdinal <= input.throughRunOrdinal
7946+
? "idle"
7947+
: providerThread.status,
7948+
pendingBackgroundTasks: [],
7949+
updatedAt: input.now,
7950+
},
79417951
});
79427952
}
79437953
});
@@ -7951,7 +7961,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
79517961
>;
79527962
readonly projection: Pick<
79537963
OrchestrationV2ThreadProjection,
7954-
"runs" | "attempts" | "nodes" | "subagents" | "runtimeRequests"
7964+
| "runs"
7965+
| "attempts"
7966+
| "nodes"
7967+
| "subagents"
7968+
| "runtimeRequests"
7969+
| "messages"
7970+
| "turnItems"
7971+
| "providerThreads"
79557972
>;
79567973
readonly providerTurn: OrchestrationV2ProviderTurn;
79577974
readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
@@ -8000,6 +8017,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
80008017
completedAt: input.now,
80018018
},
80028019
});
8020+
const providerThread = input.projection.providerThreads.find(
8021+
(candidate) => candidate.id === input.providerTurn.providerThreadId,
8022+
);
8023+
if (providerThread?.status === "active" && providerThread.lastRunOrdinal === run.ordinal) {
8024+
yield* emitEvent({
8025+
...base,
8026+
type: "provider-thread.updated",
8027+
payload: { ...providerThread, status: "idle", updatedAt: input.now },
8028+
});
8029+
}
8030+
for (const message of input.projection.messages) {
8031+
if (message.runId !== run.id || !message.streaming) continue;
8032+
yield* emitEvent({
8033+
...base,
8034+
...(message.nodeId === null ? {} : { nodeId: message.nodeId }),
8035+
type: "message.updated",
8036+
payload: { ...message, streaming: false, updatedAt: input.now },
8037+
});
8038+
}
8039+
for (const item of input.projection.turnItems) {
8040+
if (
8041+
item.runId !== run.id ||
8042+
(item.type !== "assistant_message" && item.type !== "reasoning") ||
8043+
!["pending", "running", "waiting"].includes(item.status)
8044+
)
8045+
continue;
8046+
yield* emitEvent({
8047+
...base,
8048+
...(item.nodeId === null ? {} : { nodeId: item.nodeId }),
8049+
type: "turn-item.updated",
8050+
payload: {
8051+
...item,
8052+
status: "interrupted",
8053+
streaming: false,
8054+
completedAt: input.now,
8055+
updatedAt: input.now,
8056+
},
8057+
});
8058+
}
80038059
const delegatedNodeIds = new Set(
80048060
input.projection.subagents
80058061
.filter((subagent) => subagent.origin === "app_owned")
@@ -8137,7 +8193,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
81378193
"providerThreads",
81388194
],
81398195
{
8140-
turnItemTypes: ["command_execution", "dynamic_tool", "subagent"],
8196+
turnItemTypes: [
8197+
"command_execution",
8198+
"dynamic_tool",
8199+
"subagent",
8200+
"assistant_message",
8201+
"reasoning",
8202+
],
81418203
turnItemStatuses: ["pending", "running", "waiting"],
81428204
},
81438205
),
@@ -8161,10 +8223,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
81618223
)
81628224
return;
81638225
const now = yield* DateTime.now;
8164-
if (stopped.providerTurn !== undefined) {
8226+
if (stopped.providerTurn?.status === "running" && stoppedRun.status === "running") {
8227+
const output = yield* projectionStore
8228+
.getThreadRecords(command.threadId, ["messages"], {
8229+
messageRunIds: [stoppedRun.id],
8230+
messageRoles: ["assistant"],
8231+
})
8232+
.pipe(
8233+
Effect.mapError(
8234+
(cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
8235+
),
8236+
);
81658237
yield* settleInterruptedRun({
81668238
command,
8167-
projection,
8239+
projection: { ...projection, messages: output.messages },
81688240
providerTurn: stopped.providerTurn,
81698241
events,
81708242
effects,
@@ -8214,7 +8286,13 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
82148286
// Thread-wide, like the Waiting strip: a settled thread's background
82158287
// work can belong to an earlier run than the one Stop targets.
82168288
{
8217-
turnItemTypes: ["command_execution", "dynamic_tool", "subagent"],
8289+
turnItemTypes: [
8290+
"command_execution",
8291+
"dynamic_tool",
8292+
"subagent",
8293+
"assistant_message",
8294+
"reasoning",
8295+
],
82188296
turnItemStatuses: ["pending", "running", "waiting"],
82198297
},
82208298
);

‎apps/server/src/orchestration-v2/ProviderFailure.test.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import { SqlError, classifySqliteError } from "effect/unstable/sql/SqlError";
2424
import { isStorageFullError } from "./StorageFailure.ts";
2525

2626
it("explains disk exhaustion through provider, platform and SQLite errors", () => {
27-
const diskFull = Object.assign(new Error("database or disk is full"), {
27+
const diskFull = Object.assign(new Error("SQLite operation failed"), {
2828
code: "ERR_SQLITE_ERROR",
2929
errcode: 13,
3030
});
@@ -33,12 +33,15 @@ it("explains disk exhaustion through provider, platform and SQLite errors", () =
3333
{ reason: { cause: { code: "ENOSPC" } } },
3434
new SqlError({ reason: classifySqliteError(diskFull, { operation: "execute" }) }),
3535
Cause.fail({ cause: diskFull }),
36+
Cause.combine(Cause.fail({ code: "EACCES" }), Cause.fail({ code: "ENOSPC" })),
37+
Cause.combine(Cause.fail({ code: "EACCES" }), Cause.die({ code: "ENOSPC" })),
3638
]) {
3739
assert.isTrue(isStorageFullError(cause));
3840
assert.include(makeProviderFailure({ cause }).message, "Free space on the server");
3941
}
4042
assert.isFalse(isStorageFullError({ code: "EACCES", message: "Permission denied" }));
4143
assert.isFalse(isStorageFullError(new Error("database is locked")));
44+
assert.isFalse(isStorageFullError({ code: "ERR_SQLITE_ERROR", errcode: 5 }));
4245
});
4346

4447
it("redacts credentials and URL secrets from provider failures", () => {

‎apps/server/src/orchestration-v2/ProviderTurnControlService.ts‎

Lines changed: 46 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,34 @@ export const layer: Layer.Layer<
168168
return { context, providerThread: interruptProviderThread, providerTurn, session };
169169
});
170170

171+
const awaitTerminal = (input: {
172+
readonly threadId: ThreadId;
173+
readonly providerThreadId: ProviderThreadId;
174+
readonly providerTurnId: ProviderTurnId;
175+
readonly attemptId: RunAttemptId;
176+
}) =>
177+
Effect.gen(function* () {
178+
for (let remaining = 1_000; remaining > 0; remaining -= 1) {
179+
const { providerTurn, attempt } = yield* projections.getProviderControlContext(
180+
input.threadId,
181+
input,
182+
);
183+
if (
184+
providerTurn !== undefined &&
185+
providerTurn.status !== "running" &&
186+
attempt !== undefined &&
187+
attempt.status !== "running"
188+
) {
189+
return true;
190+
}
191+
// Terminal events run on a detached ingestion fiber. Let it finish
192+
// before Stop synthesizes settlement or Restart replaces the turn.
193+
// Yield through Node so deterministic clocks cannot pin ingestion.
194+
yield* yieldToRuntime;
195+
}
196+
return false;
197+
});
198+
171199
return ProviderTurnControlServiceV2.of({
172200
interrupt: (input) =>
173201
Effect.gen(function* () {
@@ -185,6 +213,17 @@ export const layer: Layer.Layer<
185213
providerTurnId: loaded.providerTurn.id,
186214
requestRuntimeRestart: true,
187215
});
216+
if (
217+
loaded.providerTurn.status === "running" &&
218+
loaded.providerTurn.runAttemptId !== null
219+
) {
220+
// A dead native turn may acknowledge Stop without emitting a
221+
// terminal. Bound the wait so the orchestrator can recover it.
222+
yield* awaitTerminal({
223+
...input,
224+
attemptId: loaded.providerTurn.runAttemptId,
225+
});
226+
}
188227
}).pipe(
189228
Effect.mapError((cause) =>
190229
isProviderTurnControlError(cause)
@@ -222,29 +261,13 @@ export const layer: Layer.Layer<
222261
providerTurnId: loaded.providerTurn.id,
223262
});
224263

225-
for (let remaining = 1_000; remaining > 0; remaining -= 1) {
226-
const { providerTurn, attempt } = yield* projections.getProviderControlContext(
227-
input.threadId,
228-
{
229-
providerThreadId: input.providerThreadId,
230-
providerTurnId: input.providerTurnId,
231-
attemptId: input.interruptedAttemptId,
232-
},
233-
);
234-
if (
235-
providerTurn !== undefined &&
236-
providerTurn.status !== "running" &&
237-
attempt !== undefined &&
238-
attempt.status !== "running"
239-
) {
240-
return;
241-
}
242-
// Provider terminal events are projected on a detached ingestion
243-
// fiber. Yield through the Node event loop instead of sleeping on
244-
// Effect's clock so deterministic runtimes cannot deadlock a
245-
// command that is waiting for that projection.
246-
yield* yieldToRuntime;
247-
}
264+
if (
265+
yield* awaitTerminal({
266+
...input,
267+
attemptId: input.interruptedAttemptId,
268+
})
269+
)
270+
return;
248271
return yield* new ProviderTurnControlError({
249272
threadId: input.threadId,
250273
operation: "restart",

‎apps/server/src/orchestration-v2/StorageFailure.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,19 @@ export function isStorageFullError(error: unknown): boolean {
1818
if (typeof current !== "object") continue;
1919
try {
2020
if (Cause.isCause(current)) {
21-
pending.push(Cause.squash(current));
21+
for (const reason of current.reasons.slice(0, remaining)) {
22+
if (Cause.isFailReason(reason)) pending.push(reason.error);
23+
else if (Cause.isDieReason(reason)) pending.push(reason.defect);
24+
}
2225
continue;
2326
}
2427
const value = current as Record<string, unknown>;
25-
if (value.code === "ENOSPC" || value.code === "SQLITE_FULL") return true;
28+
if (
29+
value.code === "ENOSPC" ||
30+
value.code === "SQLITE_FULL" ||
31+
(value.code === "ERR_SQLITE_ERROR" && value.errcode === 13)
32+
)
33+
return true;
2634
pending.push(value.cause, value.reason, value.message);
2735
} catch {
2836
continue;

0 commit comments

Comments
 (0)