Skip to content

Commit 2ed00b7

Browse files
fix(server): restart continuations retry failed starts and respect a parent's stop
After an update the server continues the turns it cut. On a loaded host a resumed Claude session can time out registering its MCP tools before it takes the prompt; that continuation failed once and was never retried, so the restart's promise silently lapsed. A continuation that fails before the provider accepted its prompt now schedules another continuation after 1 and then 3 minutes, unless the failure is a usage, permission or validation error. The pending retry keeps a delegated child's task open, so its parent hears the real outcome, and Stop during the backoff cancels it. A parent's Stop reaches its delegated children through an effect the restart can cut short. Recovery then continued those children as if nothing happened, and they ran until the replayed stop caught up minutes later. A child's continuation now stands down when its parent's run already has a stop request, and the task settles as cancelled. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 6de3d3d commit 2ed00b7

6 files changed

Lines changed: 403 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2448,6 +2448,65 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
24482448
const task = settled.subagents.find((row) => row.id === child.taskId);
24492449
assert.equal(task?.status, "cancelled");
24502450
assert.equal(task?.completionDelivery?.state, "claimed");
2451+
2452+
// The parent's Stop committed, but the restart cut the effect that stops its children.
2453+
const stoppedChild = yield* seedRestartCancelledChild({
2454+
parentThreadId: threadId,
2455+
projectId,
2456+
parentRunId: runId,
2457+
rootNodeId,
2458+
name: "restart-parent-stopped-child",
2459+
completionWake: "always",
2460+
continuationPending: true,
2461+
now,
2462+
});
2463+
yield* (yield* EventSink.EventSinkV2).write({
2464+
commandId: CommandId.make("command:restart-declined-parent:stop"),
2465+
events: [
2466+
{
2467+
id: EventId.make("event:restart-declined-parent:interrupt-request"),
2468+
type: "turn-item.updated",
2469+
threadId,
2470+
runId,
2471+
nodeId: rootNodeId,
2472+
providerInstanceId: modelSelection.instanceId,
2473+
occurredAt: now,
2474+
payload: {
2475+
id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"),
2476+
type: "run_interrupt_request",
2477+
threadId,
2478+
runId,
2479+
nodeId: rootNodeId,
2480+
providerThreadId: null,
2481+
providerTurnId: null,
2482+
nativeItemRef: null,
2483+
parentItemId: null,
2484+
ordinal: 90,
2485+
status: "completed",
2486+
title: "Interrupt requested",
2487+
startedAt: now,
2488+
completedAt: now,
2489+
updatedAt: now,
2490+
message: "Stopped by user",
2491+
},
2492+
},
2493+
],
2494+
});
2495+
yield* continueRestartedRun({
2496+
threadId: stoppedChild.childThreadId,
2497+
sourceRunId: stoppedChild.childRunId,
2498+
}).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })));
2499+
const afterStop = yield* orchestrator.getThreadProjection(threadId);
2500+
assert.equal(
2501+
afterStop.subagents.find((row) => row.id === stoppedChild.taskId)?.status,
2502+
"cancelled",
2503+
);
2504+
assert.deepEqual(
2505+
(yield* orchestrator.getThreadProjection(stoppedChild.childThreadId)).runs.map(
2506+
(run) => run.id,
2507+
),
2508+
[stoppedChild.childRunId],
2509+
);
24512510
}),
24522511
);
24532512

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
107107
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
108108
import {
109109
cancelledBeforeStart,
110+
continuationFailedBeforePrompt,
111+
continuationStartRetryDelayMs,
110112
isRestartContinuationSource,
111113
runOutcome,
112114
} from "./RestartBackgroundNote.ts";
@@ -2195,6 +2197,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
21952197
),
21962198
);
21972199

2200+
// Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
2201+
const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
2202+
threadId: ThreadId,
2203+
runId: RunId,
2204+
) {
2205+
const projection = yield* projectionStore.getThreadRecords(
2206+
threadId,
2207+
["runs", "providerTurns", "turnItems"],
2208+
{ turnItemTypes: ["error"], turnItemRunIds: [runId] },
2209+
);
2210+
const run = projection.runs.find((candidate) => candidate.id === runId);
2211+
if (run === undefined) return;
2212+
const delayMs = continuationStartRetryDelayMs(run, projection);
2213+
if (delayMs === undefined) return;
2214+
const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
2215+
yield* writeSystemEvents(
2216+
[],
2217+
[
2218+
{
2219+
id: `effect:restart-continuation:${run.id}`,
2220+
commandId,
2221+
threadId,
2222+
request: { type: "provider-runtime.continue", sourceRunId: run.id },
2223+
availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
2224+
},
2225+
],
2226+
);
2227+
});
2228+
21982229
// A hold waits for a person to review what they queued. Nobody reviews messages from agents,
21992230
// the server or scheduled tasks, so a later completed run releases those unless it was stopped.
22002231
const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
@@ -5166,7 +5197,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
51665197
const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
51675198
if (
51685199
!source ||
5169-
!isRestartContinuationSource(source, projection.providerTurns) ||
5200+
!(
5201+
isRestartContinuationSource(source, projection.providerTurns) ||
5202+
continuationFailedBeforePrompt(source, projection.providerTurns)
5203+
) ||
51705204
projection.thread.archivedAt !== null ||
51715205
projection.thread.deletedAt !== null ||
51725206
projection.thread.providerInstanceId !== source.providerInstanceId ||
@@ -10404,7 +10438,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1040410438
// A resumed queue can have a lower ordinal than a pending continuation source.
1040510439
// Stop must reach those sources even when interrupting the resumed run succeeds.
1040610440
if (
10407-
run.status === "cancelled" &&
10441+
(run.status === "cancelled" || run.status === "failed") &&
1040810442
(yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
1040910443
) {
1041010444
yield* markStoppedRun({ ...input, thread, run });
@@ -13114,6 +13148,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1311413148
// while writing child events, so nesting parent inside child would invert the order.
1311513149
// The keyed semaphores are neither reentrant nor deadlock-aware.
1311613150
if (delegatedTaskTerminalStatus(run.status) !== null) {
13151+
// Before the parent hears of the failure: a pending continuation keeps the task open.
13152+
if (run.status === "failed" && run.restartContinuationOfRunId !== undefined) {
13153+
yield* threadDispatch.withLock(threadId, retryContinuationStart(threadId, run.id));
13154+
}
1311713155
yield* threadDispatch.withLock(
1311813156
threadId,
1311913157
finalizeDelegatedCompletionDelivery(threadId, run.id),

‎apps/server/src/orchestration-v2/RestartBackgroundNote.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import type {
66
OrchestrationV2RunAttempt,
77
OrchestrationV2TurnItem,
88
} from "@t3tools/contracts";
9-
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
9+
import { latestRootProviderFailure, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
1010

1111
type Work = OrchestrationV2RestartCancelledBackgroundWork;
1212
type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
@@ -181,6 +181,50 @@ export function cancelledBeforeStart(
181181
return run.status === "cancelled" && neverStarted(run);
182182
}
183183

184+
/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
185+
export function continuationFailedBeforePrompt(
186+
run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
187+
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
188+
): boolean {
189+
return (
190+
run.status === "failed" &&
191+
run.restartContinuationOfRunId !== undefined &&
192+
!providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
193+
);
194+
}
195+
196+
const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000];
197+
198+
/** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */
199+
export function continuationStartRetryDelayMs(
200+
run: OrchestrationV2Run,
201+
projection: {
202+
readonly runs: ReadonlyArray<OrchestrationV2Run>;
203+
readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
204+
readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
205+
},
206+
): number | undefined {
207+
if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return;
208+
// These fail the same way however often they start.
209+
const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class;
210+
if (
211+
failureClass === "usage_limit" ||
212+
failureClass === "permission_error" ||
213+
failureClass === "validation_error"
214+
)
215+
return;
216+
let failedStarts = 0;
217+
for (
218+
let current: OrchestrationV2Run | undefined = run;
219+
current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns);
220+
current = projection.runs.find(
221+
(candidate) => candidate.id === current?.restartContinuationOfRunId,
222+
)
223+
)
224+
failedStarts += 1;
225+
return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1];
226+
}
227+
184228
export function isRestartContinuationSource(
185229
source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
186230
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,

‎apps/server/src/orchestration-v2/RestartContinuation.test.ts‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
2020
import * as Layer from "effect/Layer";
2121
import * as ServerSettings from "../serverSettings.ts";
2222
import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
23+
import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
2324
import * as ThreadManagementService from "./ThreadManagementService.ts";
2425
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
2526
import * as ProjectionStore from "./ProjectionStore.ts";
@@ -46,6 +47,8 @@ function makeProjection() {
4647
providerInstanceId: instanceId,
4748
archivedAt: null,
4849
deletedAt: null,
50+
lineage: { parentThreadId: null, relationshipToParent: null },
51+
forkedFrom: null,
4952
},
5053
runs: [
5154
{
@@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
839842
}),
840843
);
841844

842-
const continuationDispatches = (projection: OrchestrationV2ThreadProjection) =>
845+
const continuationDispatches = (
846+
projection: OrchestrationV2ThreadProjection,
847+
sourceRunId: RunId = runId,
848+
) =>
843849
Effect.gen(function* () {
844850
const dispatches: Array<Extract<OrchestrationV2Command, { type: "message.dispatch" }>> = [];
845-
yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
851+
yield* continueRestartedRun({ threadId, sourceRunId }).pipe(
846852
Effect.provide(
847853
Layer.merge(
848854
Layer.mock(ThreadManagementService.ThreadManagementService)({
@@ -943,6 +949,72 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
943949
}),
944950
);
945951

952+
it("backs off twice from a continuation the provider never took, then lets it fail", () => {
953+
const base = makeProjection();
954+
const failedStart = (id: string, of: RunId) => ({
955+
...base.runs[0]!,
956+
id: RunId.make(id),
957+
activeAttemptId: RunAttemptId.make(`attempt:${id}`),
958+
status: "failed" as const,
959+
restartContinuationOfRunId: of,
960+
});
961+
const first = failedStart("run:retry-1", runId);
962+
const second = failedStart("run:retry-2", first.id);
963+
const third = failedStart("run:retry-3", second.id);
964+
const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) =>
965+
({
966+
runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third],
967+
providerTurns,
968+
turnItems: [first, second, third].map((run) => ({
969+
id: `error:${run.id}`,
970+
type: "error",
971+
status: "failed",
972+
runId: run.id,
973+
nodeId: run.rootNodeId,
974+
ordinal: 1,
975+
updatedAt: now,
976+
failure: { class: failureClass, message: "Failed.", code: null, retryable: null },
977+
})),
978+
}) as unknown as Parameters<typeof continuationStartRetryDelayMs>[1];
979+
assert.deepEqual(
980+
[first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())),
981+
[60_000, 180_000, undefined],
982+
);
983+
for (const failureClass of ["usage_limit", "permission_error", "validation_error"])
984+
assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass)));
985+
// Once the provider took the prompt, starting again would repeat the turn.
986+
assert.isUndefined(
987+
continuationStartRetryDelayMs(
988+
first,
989+
projection("provider_error", [
990+
{ ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" },
991+
] as never),
992+
),
993+
);
994+
});
995+
996+
it.effect("continues again after a continuation fails before its prompt", () =>
997+
Effect.gen(function* () {
998+
const cut = cutMidTurn();
999+
const failedStart = {
1000+
...cut.runs[0]!,
1001+
id: RunId.make("run:failed-start"),
1002+
ordinal: 2,
1003+
activeAttemptId: RunAttemptId.make("attempt:failed-start"),
1004+
status: "failed",
1005+
restartContinuationOfRunId: runId,
1006+
};
1007+
const dispatches = yield* continuationDispatches(
1008+
{ ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection,
1009+
failedStart.id,
1010+
);
1011+
assert.deepEqual(
1012+
dispatches.map((command) => command.restartContinuationOfRunId),
1013+
[failedStart.id],
1014+
);
1015+
}),
1016+
);
1017+
9461018
it.effect("does not continue a run the user asked to stop before the restart", () =>
9471019
Effect.gen(function* () {
9481020
const texts = yield* continuationTexts({

‎apps/server/src/orchestration-v2/RestartContinuation.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import {
1919
awaitingProviderThreadId,
2020
cancelledRosterTaskWork,
2121
cancelledTurnItemWork,
22+
continuationFailedBeforePrompt,
2223
isAwaitedWork,
2324
isRestartContinuationSource,
2425
cancelledBeforeStart,
@@ -200,7 +201,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
200201
const source = projection.runs.find((run) => run.id === input.sourceRunId);
201202
if (
202203
!source ||
203-
!isRestartContinuationSource(source, projection.providerTurns) ||
204+
!(
205+
isRestartContinuationSource(source, projection.providerTurns) ||
206+
continuationFailedBeforePrompt(source, projection.providerTurns)
207+
) ||
204208
restartContinuationBlocked(
205209
projection,
206210
source,
@@ -255,6 +259,27 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
255259
)
256260
)
257261
return;
262+
// A parent's Stop reaches its delegated children through an effect a restart can cut short.
263+
const { lineage, forkedFrom } = projection.thread;
264+
const taskId =
265+
source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
266+
if (
267+
lineage.relationshipToParent === "subagent" &&
268+
lineage.parentThreadId !== null &&
269+
taskId !== undefined
270+
) {
271+
const parentRunId = (yield* threads.getThreadRecords(lineage.parentThreadId, ["subagents"], {
272+
subagentIds: [taskId],
273+
})).subagents.find((task) => task.id === taskId)?.runId;
274+
if (
275+
parentRunId != null &&
276+
(yield* threads.getThreadRecords(lineage.parentThreadId, ["turnItems"], {
277+
turnItemTypes: ["run_interrupt_request"],
278+
turnItemRunIds: [parentRunId],
279+
})).turnItems.length > 0
280+
)
281+
return;
282+
}
258283
const sourceMessage = sourceRecords.messages.find(
259284
(message) => message.id === source.userMessageId,
260285
);

0 commit comments

Comments
 (0)