Skip to content

Commit f9a8136

Browse files
fix(server): restart continuations retry failed starts and respect a parent's stop
After an update the server continues the turns it cut. On a loaded host a resumed Claude session can time out registering its MCP tools before it takes the prompt; that continuation failed once and was never retried, so the restart's promise silently lapsed. A continuation that fails before the provider accepted its prompt now schedules another continuation after 1 and then 3 minutes, unless the failure is a usage, permission or validation error. The pending retry keeps a delegated child's task open, so its parent hears the real outcome, and Stop during the backoff cancels it. A parent's Stop reaches its delegated children through an effect the restart can cut short. Recovery then continued those children as if nothing happened, and they ran until the replayed stop caught up minutes later. A child's continuation now stands down when its parent's run already has a stop request, and the task settles as cancelled. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 4f97c80 commit f9a8136

6 files changed

Lines changed: 464 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2448,6 +2448,65 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
24482448
const task = settled.subagents.find((row) => row.id === child.taskId);
24492449
assert.equal(task?.status, "cancelled");
24502450
assert.equal(task?.completionDelivery?.state, "claimed");
2451+
2452+
// The parent's Stop committed, but the restart cut the effect that stops its children.
2453+
const stoppedChild = yield* seedRestartCancelledChild({
2454+
parentThreadId: threadId,
2455+
projectId,
2456+
parentRunId: runId,
2457+
rootNodeId,
2458+
name: "restart-parent-stopped-child",
2459+
completionWake: "always",
2460+
continuationPending: true,
2461+
now,
2462+
});
2463+
yield* (yield* EventSink.EventSinkV2).write({
2464+
commandId: CommandId.make("command:restart-declined-parent:stop"),
2465+
events: [
2466+
{
2467+
id: EventId.make("event:restart-declined-parent:interrupt-request"),
2468+
type: "turn-item.updated",
2469+
threadId,
2470+
runId,
2471+
nodeId: rootNodeId,
2472+
providerInstanceId: modelSelection.instanceId,
2473+
occurredAt: now,
2474+
payload: {
2475+
id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"),
2476+
type: "run_interrupt_request",
2477+
threadId,
2478+
runId,
2479+
nodeId: rootNodeId,
2480+
providerThreadId: null,
2481+
providerTurnId: null,
2482+
nativeItemRef: null,
2483+
parentItemId: null,
2484+
ordinal: 90,
2485+
status: "completed",
2486+
title: "Interrupt requested",
2487+
startedAt: now,
2488+
completedAt: now,
2489+
updatedAt: now,
2490+
message: "Stopped by user",
2491+
},
2492+
},
2493+
],
2494+
});
2495+
yield* continueRestartedRun({
2496+
threadId: stoppedChild.childThreadId,
2497+
sourceRunId: stoppedChild.childRunId,
2498+
}).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })));
2499+
const afterStop = yield* orchestrator.getThreadProjection(threadId);
2500+
assert.equal(
2501+
afterStop.subagents.find((row) => row.id === stoppedChild.taskId)?.status,
2502+
"cancelled",
2503+
);
2504+
assert.deepEqual(
2505+
(yield* orchestrator.getThreadProjection(stoppedChild.childThreadId)).runs.map(
2506+
(run) => run.id,
2507+
),
2508+
[stoppedChild.childRunId],
2509+
);
24512510
}),
24522511
);
24532512

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
107107
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
108108
import {
109109
cancelledBeforeStart,
110+
continuationFailedBeforePrompt,
111+
continuationStartRetryDelayMs,
110112
isRestartContinuationSource,
111113
runOutcome,
112114
} from "./RestartBackgroundNote.ts";
@@ -2209,6 +2211,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
22092211
),
22102212
);
22112213

2214+
// Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
2215+
const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
2216+
threadId: ThreadId,
2217+
runId: RunId,
2218+
) {
2219+
const projection = yield* projectionStore.getThreadRecords(
2220+
threadId,
2221+
["runs", "providerTurns", "turnItems"],
2222+
{ turnItemTypes: ["error"], turnItemRunIds: [runId] },
2223+
);
2224+
const run = projection.runs.find((candidate) => candidate.id === runId);
2225+
if (run === undefined) return;
2226+
const delayMs = continuationStartRetryDelayMs(run, projection);
2227+
if (delayMs === undefined) return;
2228+
const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
2229+
yield* writeSystemEvents(
2230+
[],
2231+
[
2232+
{
2233+
id: `effect:restart-continuation:${run.id}`,
2234+
commandId,
2235+
threadId,
2236+
request: { type: "provider-runtime.continue", sourceRunId: run.id },
2237+
availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
2238+
},
2239+
],
2240+
);
2241+
});
2242+
22122243
// A hold waits for a person to review what they queued. Nobody reviews messages from agents,
22132244
// the server or scheduled tasks, so a later completed run releases those unless it was stopped.
22142245
const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
@@ -5188,7 +5219,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
51885219
const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
51895220
if (
51905221
!source ||
5191-
!isRestartContinuationSource(source, projection.providerTurns) ||
5222+
!(
5223+
isRestartContinuationSource(source, projection.providerTurns) ||
5224+
continuationFailedBeforePrompt(source, projection.providerTurns)
5225+
) ||
51925226
projection.thread.archivedAt !== null ||
51935227
projection.thread.deletedAt !== null ||
51945228
projection.thread.providerInstanceId !== source.providerInstanceId ||
@@ -10419,7 +10453,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1041910453
// A resumed queue can have a lower ordinal than a pending continuation source.
1042010454
// Stop must reach those sources even when interrupting the resumed run succeeds.
1042110455
if (
10422-
run.status === "cancelled" &&
10456+
(run.status === "cancelled" ||
10457+
(run.status === "failed" && run.restartContinuationOfRunId !== undefined)) &&
1042310458
(yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
1042410459
) {
1042510460
yield* markStoppedRun({ ...input, thread, run });
@@ -13129,6 +13164,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1312913164
// while writing child events, so nesting parent inside child would invert the order.
1313013165
// The keyed semaphores are neither reentrant nor deadlock-aware.
1313113166
if (delegatedTaskTerminalStatus(run.status) !== null) {
13167+
// Before the parent hears of the failure: a pending continuation keeps the task open.
13168+
if (run.status === "failed" && run.restartContinuationOfRunId !== undefined) {
13169+
yield* threadDispatch.withLock(threadId, retryContinuationStart(threadId, run.id)).pipe(
13170+
Effect.catchCause((cause) =>
13171+
Effect.logWarning("Failed to schedule a restart continuation retry", {
13172+
threadId,
13173+
runId: run.id,
13174+
cause,
13175+
}),
13176+
),
13177+
);
13178+
}
1313213179
yield* threadDispatch.withLock(
1313313180
threadId,
1313413181
finalizeDelegatedCompletionDelivery(threadId, run.id),

‎apps/server/src/orchestration-v2/RestartBackgroundNote.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import type {
66
OrchestrationV2RunAttempt,
77
OrchestrationV2TurnItem,
88
} from "@t3tools/contracts";
9-
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
9+
import { latestRootProviderFailure, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
1010

1111
type Work = OrchestrationV2RestartCancelledBackgroundWork;
1212
type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
@@ -181,6 +181,50 @@ export function cancelledBeforeStart(
181181
return run.status === "cancelled" && neverStarted(run);
182182
}
183183

184+
/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
185+
export function continuationFailedBeforePrompt(
186+
run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
187+
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
188+
): boolean {
189+
return (
190+
run.status === "failed" &&
191+
run.restartContinuationOfRunId !== undefined &&
192+
!providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
193+
);
194+
}
195+
196+
const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000];
197+
198+
/** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */
199+
export function continuationStartRetryDelayMs(
200+
run: OrchestrationV2Run,
201+
projection: {
202+
readonly runs: ReadonlyArray<OrchestrationV2Run>;
203+
readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
204+
readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
205+
},
206+
): number | undefined {
207+
if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return;
208+
// These fail the same way however often they start.
209+
const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class;
210+
if (
211+
failureClass === "usage_limit" ||
212+
failureClass === "permission_error" ||
213+
failureClass === "validation_error"
214+
)
215+
return;
216+
let failedStarts = 0;
217+
for (
218+
let current: OrchestrationV2Run | undefined = run;
219+
current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns);
220+
current = projection.runs.find(
221+
(candidate) => candidate.id === current?.restartContinuationOfRunId,
222+
)
223+
)
224+
failedStarts += 1;
225+
return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1];
226+
}
227+
184228
export function isRestartContinuationSource(
185229
source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
186230
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,

‎apps/server/src/orchestration-v2/RestartContinuation.test.ts‎

Lines changed: 120 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
2020
import * as Layer from "effect/Layer";
2121
import * as ServerSettings from "../serverSettings.ts";
2222
import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
23+
import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
2324
import * as ThreadManagementService from "./ThreadManagementService.ts";
2425
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
2526
import * as ProjectionStore from "./ProjectionStore.ts";
@@ -46,6 +47,8 @@ function makeProjection() {
4647
providerInstanceId: instanceId,
4748
archivedAt: null,
4849
deletedAt: null,
50+
lineage: { parentThreadId: null, relationshipToParent: null },
51+
forkedFrom: null,
4952
},
5053
runs: [
5154
{
@@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
839842
}),
840843
);
841844

842-
const continuationDispatches = (projection: OrchestrationV2ThreadProjection) =>
845+
const continuationDispatches = (
846+
projection: OrchestrationV2ThreadProjection,
847+
sourceRunId: RunId = runId,
848+
) =>
843849
Effect.gen(function* () {
844850
const dispatches: Array<Extract<OrchestrationV2Command, { type: "message.dispatch" }>> = [];
845-
yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
851+
yield* continueRestartedRun({ threadId, sourceRunId }).pipe(
846852
Effect.provide(
847853
Layer.merge(
848854
Layer.mock(ThreadManagementService.ThreadManagementService)({
@@ -943,6 +949,118 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
943949
}),
944950
);
945951

952+
it("backs off twice from a continuation the provider never took, then lets it fail", () => {
953+
const base = makeProjection();
954+
const failedStart = (id: string, of: RunId) => ({
955+
...base.runs[0]!,
956+
id: RunId.make(id),
957+
activeAttemptId: RunAttemptId.make(`attempt:${id}`),
958+
status: "failed" as const,
959+
restartContinuationOfRunId: of,
960+
});
961+
const first = failedStart("run:retry-1", runId);
962+
const second = failedStart("run:retry-2", first.id);
963+
const third = failedStart("run:retry-3", second.id);
964+
const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) =>
965+
({
966+
runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third],
967+
providerTurns,
968+
turnItems: [first, second, third].map((run) => ({
969+
id: `error:${run.id}`,
970+
type: "error",
971+
status: "failed",
972+
runId: run.id,
973+
nodeId: run.rootNodeId,
974+
ordinal: 1,
975+
updatedAt: now,
976+
failure: { class: failureClass, message: "Failed.", code: null, retryable: null },
977+
})),
978+
}) as unknown as Parameters<typeof continuationStartRetryDelayMs>[1];
979+
assert.deepEqual(
980+
[first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())),
981+
[60_000, 180_000, undefined],
982+
);
983+
for (const failureClass of ["usage_limit", "permission_error", "validation_error"])
984+
assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass)));
985+
// Once the provider took the prompt, starting again would repeat the turn.
986+
assert.isUndefined(
987+
continuationStartRetryDelayMs(
988+
first,
989+
projection("provider_error", [
990+
{ ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" },
991+
] as never),
992+
),
993+
);
994+
});
995+
996+
it.effect("continues again after a continuation fails before its prompt", () =>
997+
Effect.gen(function* () {
998+
const cut = cutMidTurn();
999+
const failedStart = {
1000+
...cut.runs[0]!,
1001+
id: RunId.make("run:failed-start"),
1002+
ordinal: 2,
1003+
activeAttemptId: RunAttemptId.make("attempt:failed-start"),
1004+
status: "failed",
1005+
restartContinuationOfRunId: runId,
1006+
};
1007+
const dispatches = yield* continuationDispatches(
1008+
{ ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection,
1009+
failedStart.id,
1010+
);
1011+
assert.deepEqual(
1012+
dispatches.map((command) => command.restartContinuationOfRunId),
1013+
[failedStart.id],
1014+
);
1015+
}),
1016+
);
1017+
1018+
it.effect.each([
1019+
["running", null, 0],
1020+
["completed", "Done.", 1],
1021+
] as const)(
1022+
"a parent's Stop on the task's run blocks the child only while the task is %s",
1023+
([status, result, expected]) =>
1024+
Effect.gen(function* () {
1025+
const parentThreadId = ThreadId.make("thread:parent");
1026+
const parentRunId = RunId.make("run:parent");
1027+
const taskId = NodeId.make("node:task");
1028+
const cut = cutMidTurn();
1029+
const child = {
1030+
...cut,
1031+
thread: {
1032+
...cut.thread,
1033+
lineage: { parentThreadId, relationshipToParent: "subagent" },
1034+
forkedFrom: { type: "node", nodeId: taskId },
1035+
},
1036+
} as unknown as OrchestrationV2ThreadProjection;
1037+
const parent = {
1038+
thread: { id: parentThreadId },
1039+
subagents: [{ id: taskId, runId: parentRunId, origin: "app_owned", status, result }],
1040+
turnItems: [{ id: "turn-item:stop", runId: parentRunId, type: "run_interrupt_request" }],
1041+
};
1042+
let dispatched = 0;
1043+
yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
1044+
Effect.provide(
1045+
Layer.merge(
1046+
Layer.mock(ThreadManagementService.ThreadManagementService)({
1047+
getThreadRecords: (id) =>
1048+
Effect.succeed((id === parentThreadId ? parent : child) as never),
1049+
recoverDelegatedTask: () => Effect.void,
1050+
dispatch: () =>
1051+
Effect.sync(() => {
1052+
dispatched += 1;
1053+
return {} as never;
1054+
}),
1055+
}),
1056+
ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1057+
),
1058+
),
1059+
);
1060+
assert.equal(dispatched, expected);
1061+
}),
1062+
);
1063+
9461064
it.effect("does not continue a run the user asked to stop before the restart", () =>
9471065
Effect.gen(function* () {
9481066
const texts = yield* continuationTexts({

0 commit comments

Comments
 (0)