Skip to content

Commit 2a7b36d

Browse files
committed
fix(orchestration-v2): recover runs after disk space stalls
- Retry terminal writes after disk exhaustion and explain storage failures - Settle stopped runs when their provider turn is already gone
1 parent f90b77d commit 2a7b36d

9 files changed

Lines changed: 629 additions & 28 deletions

‎apps/server/src/orchestration-v2/EffectWorker.test.ts‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,14 @@ function makeExecutorLayer(input: {
8181
readonly failFirstStart?: Ref.Ref<boolean>;
8282
readonly threads?: Partial<ThreadManagementService.ThreadManagementService["Service"]>;
8383
readonly continueAfterRestart?: boolean;
84+
readonly interrupt?: ProviderTurnControlService.ProviderTurnControlServiceV2Shape["interrupt"];
8485
}) {
8586
const record = (event: string) => Ref.update(input.events, (events) => [...events, event]);
8687
const dependencies = Layer.mergeAll(
8788
Layer.succeed(
8889
ProviderTurnControlService.ProviderTurnControlServiceV2,
8990
ProviderTurnControlService.ProviderTurnControlServiceV2.of({
90-
interrupt: () => Effect.void,
91+
interrupt: input.interrupt ?? (() => Effect.void),
9192
steer: () => Effect.void,
9293
interruptAndAwaitTerminal: (request) =>
9394
record(
@@ -193,6 +194,42 @@ it("does not retry pure interrupt races where the turn is already gone", () => {
193194
);
194195
});
195196

197+
it.effect("settles a stopped run when its adapter has already lost the native turn", () =>
198+
Effect.gen(function* () {
199+
const now = yield* DateTime.now;
200+
const events = yield* Ref.make<ReadonlyArray<string>>([]);
201+
const layer = makeExecutorLayer({
202+
events,
203+
interrupt: () =>
204+
new ProviderTurnControlService.ProviderTurnControlError({
205+
threadId,
206+
operation: "interrupt",
207+
providerTurnId,
208+
cause: "Provider turn is not active.",
209+
}),
210+
threads: {
211+
dispatch: (command) =>
212+
Ref.update(events, (current) => [...current, command.type]).pipe(
213+
Effect.as({ sequence: 1, storedEvents: [] }),
214+
),
215+
},
216+
});
217+
yield* Effect.gen(function* () {
218+
const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
219+
yield* executor.execute({
220+
...restartEffect(now, { type: "detach" }),
221+
request: {
222+
type: "provider-turn.interrupt",
223+
providerSessionId: oldSessionId,
224+
providerThreadId,
225+
providerTurnId,
226+
},
227+
});
228+
}).pipe(Effect.provide(layer));
229+
assert.deepEqual(yield* Ref.get(events), ["thread.background-work.settle"]);
230+
}),
231+
);
232+
196233
it.effect("requeues a claim when a pre-execution worker check fails", () =>
197234
Effect.gen(function* () {
198235
const now = DateTime.formatIso(yield* DateTime.now);

‎apps/server/src/orchestration-v2/EffectWorker.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,14 @@ export const executorLayer: Layer.Layer<
171171
providerTurnId: effect.request.providerTurnId,
172172
})
173173
.pipe(
174+
Effect.catch((cause) =>
175+
isNonRetryableProviderTurnControlFailure(
176+
effect.request.type,
177+
Cause.pretty(Cause.fail(cause)),
178+
)
179+
? Effect.void
180+
: Effect.fail(cause),
181+
),
174182
// The provider has stopped what it still ran and reported it.
175183
// Whatever the thread still shows on that provider thread is
176184
// work no process will report on, so the Stop ends it too.

‎apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts‎

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,230 @@ const testLayer = Layer.mergeAll(
4949
),
5050
);
5151

52+
it.effect.each(["missing-session", "returned-interrupt", "superseded-attempt"] as const)(
53+
"Stop recovers a stalled run safely after %s",
54+
(scenario) =>
55+
Effect.gen(function* () {
56+
const orchestrator = yield* Orchestrator.OrchestratorV2;
57+
const projections = yield* ProjectionStore.ProjectionStoreV2;
58+
const now = yield* DateTime.now;
59+
const threadId = ThreadId.make(`thread:stalled:${scenario}`);
60+
const runId = RunId.make(`run:stalled:${scenario}`);
61+
const nodeId = NodeId.make(`node:stalled:${scenario}`);
62+
const attemptId = RunAttemptId.make(`attempt:stalled:${scenario}`);
63+
const providerThreadId = ProviderThreadId.make(`provider-thread:stalled:${scenario}`);
64+
const providerTurnId = ProviderTurnId.make(`provider-turn:stalled:${scenario}`);
65+
yield* orchestrator.dispatch({
66+
type: "thread.create",
67+
commandId: CommandId.make(`create:${scenario}`),
68+
threadId,
69+
projectId: ProjectId.make(`project:stalled:${scenario}`),
70+
title: "Stalled run",
71+
modelSelection,
72+
runtimeMode: "full-access",
73+
interactionMode: "default",
74+
branch: null,
75+
worktreePath: null,
76+
createdBy: "user",
77+
creationSource: "web",
78+
});
79+
yield* projections.apply({
80+
id: EventId.make(`${scenario}:thread`),
81+
type: "provider-thread.updated",
82+
threadId,
83+
occurredAt: now,
84+
payload: {
85+
id: providerThreadId,
86+
driver: adapter.driver,
87+
providerInstanceId: instanceId,
88+
providerSessionId: ProviderSessionId.make("provider-session:gone"),
89+
appThreadId: threadId,
90+
ownerNodeId: nodeId,
91+
nativeThreadRef: null,
92+
nativeConversationHeadRef: null,
93+
status: "active",
94+
firstRunOrdinal: 1,
95+
lastRunOrdinal: 1,
96+
handoffIds: [],
97+
forkedFrom: null,
98+
createdAt: now,
99+
updatedAt: now,
100+
},
101+
});
102+
const run = {
103+
id: runId,
104+
threadId,
105+
ordinal: 1,
106+
providerInstanceId: instanceId,
107+
modelSelection,
108+
providerThreadId,
109+
userMessageId: MessageId.make(`${scenario}:user`),
110+
rootNodeId: nodeId,
111+
activeAttemptId: attemptId,
112+
status: "running" as const,
113+
requestedAt: now,
114+
startedAt: now,
115+
completedAt: null,
116+
checkpointId: null,
117+
contextHandoffId: null,
118+
};
119+
yield* projections.apply({
120+
id: EventId.make(`${scenario}:run`),
121+
type: "run.created",
122+
threadId,
123+
occurredAt: now,
124+
payload: run,
125+
});
126+
yield* projections.apply({
127+
id: EventId.make(`${scenario}:attempt`),
128+
type: "run-attempt.created",
129+
threadId,
130+
occurredAt: now,
131+
payload: {
132+
id: attemptId,
133+
runId,
134+
attemptOrdinal: 1,
135+
rootNodeId: nodeId,
136+
providerInstanceId: instanceId,
137+
providerThreadId,
138+
providerTurnId,
139+
reason: "initial",
140+
status: "running",
141+
startedAt: now,
142+
completedAt: null,
143+
},
144+
});
145+
yield* projections.apply({
146+
id: EventId.make(`${scenario}:turn`),
147+
type: "provider-turn.updated",
148+
threadId,
149+
occurredAt: now,
150+
payload: {
151+
id: providerTurnId,
152+
providerThreadId,
153+
nodeId,
154+
runAttemptId: attemptId,
155+
nativeTurnRef: null,
156+
ordinal: 1,
157+
status: "running",
158+
startedAt: now,
159+
completedAt: null,
160+
},
161+
});
162+
yield* projections.apply({
163+
id: EventId.make(`${scenario}:node`),
164+
type: "node.updated",
165+
threadId,
166+
occurredAt: now,
167+
payload: {
168+
id: nodeId,
169+
threadId,
170+
runId,
171+
parentNodeId: null,
172+
rootNodeId: nodeId,
173+
kind: "root_turn",
174+
status: "running",
175+
countsForRun: true,
176+
providerThreadId,
177+
providerTurnId,
178+
nativeItemRef: null,
179+
runtimeRequestId: null,
180+
checkpointScopeId: null,
181+
startedAt: now,
182+
completedAt: null,
183+
},
184+
});
185+
yield* projections.apply({
186+
id: EventId.make(`${scenario}:item`),
187+
type: "turn-item.updated",
188+
threadId,
189+
runId,
190+
occurredAt: now,
191+
payload: {
192+
id: TurnItemId.make(`${scenario}:tool`),
193+
threadId,
194+
runId,
195+
nodeId,
196+
providerThreadId,
197+
providerTurnId,
198+
nativeItemRef: null,
199+
parentItemId: null,
200+
ordinal: 1,
201+
status: "running",
202+
title: "Tool",
203+
startedAt: now,
204+
completedAt: null,
205+
updatedAt: now,
206+
type: "command_execution",
207+
input: "test",
208+
},
209+
});
210+
yield* projections.apply({
211+
id: EventId.make(`${scenario}:request`),
212+
type: "runtime-request.updated",
213+
threadId,
214+
occurredAt: now,
215+
payload: {
216+
id: RuntimeRequestId.make(`${scenario}:request`),
217+
nodeId,
218+
providerTurnId,
219+
nativeRequestRef: null,
220+
kind: "user_input",
221+
status: "pending",
222+
responseCapability: {
223+
type: "live",
224+
providerSessionId: ProviderSessionId.make("provider-session:gone"),
225+
},
226+
createdAt: now,
227+
resolvedAt: null,
228+
},
229+
});
230+
if (scenario === "superseded-attempt") {
231+
yield* projections.apply({
232+
id: EventId.make(`${scenario}:new-attempt`),
233+
type: "run.updated",
234+
threadId,
235+
occurredAt: now,
236+
payload: { ...run, activeAttemptId: RunAttemptId.make("attempt:newer") },
237+
});
238+
}
239+
const command =
240+
scenario === "missing-session"
241+
? {
242+
type: "run.interrupt" as const,
243+
commandId: CommandId.make(`stop:${scenario}`),
244+
threadId,
245+
runId,
246+
}
247+
: {
248+
type: "thread.background-work.settle" as const,
249+
commandId: CommandId.make(`stop:${scenario}`),
250+
threadId,
251+
providerThreadId,
252+
providerTurnId,
253+
};
254+
const first = yield* orchestrator.dispatch(command);
255+
const after = yield* projections.getThreadProjection(threadId);
256+
const interrupted = scenario !== "superseded-attempt";
257+
assert.equal(after.runs[0]?.status, interrupted ? "interrupted" : "running");
258+
assert.equal(after.attempts[0]?.status, interrupted ? "interrupted" : "running");
259+
assert.equal(after.providerTurns[0]?.status, interrupted ? "interrupted" : "running");
260+
assert.equal(after.nodes[0]?.status, interrupted ? "interrupted" : "running");
261+
assert.equal(
262+
after.turnItems.find((item) => item.type === "command_execution")?.status,
263+
interrupted ? "interrupted" : "running",
264+
);
265+
assert.equal(after.runtimeRequests[0]?.status, interrupted ? "cancelled" : "pending");
266+
assert.equal(
267+
after.turnItems.filter((item) => item.type === "run_interrupt_result").length,
268+
interrupted ? 1 : 0,
269+
);
270+
const replay = yield* orchestrator.dispatch(command);
271+
assert.equal(replay.sequence, first.sequence);
272+
assert.deepEqual(replay.storedEvents, first.storedEvents);
273+
}).pipe(Effect.provide(testLayer)),
274+
);
275+
52276
it.effect(
53277
"dispatches metadata, queue resume and request controls without hydrating unrelated history",
54278
() =>

0 commit comments

Comments
 (0)