Skip to content

Commit 36dac68

Browse files
fix(server): restart continuations retry failed starts and respect a parent's stop (#27222)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 8ab098a commit 36dac68

6 files changed

Lines changed: 464 additions & 6 deletions

File tree

‎apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2448,6 +2448,65 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
24482448
const task = settled.subagents.find((row) => row.id === child.taskId);
24492449
assert.equal(task?.status, "cancelled");
24502450
assert.equal(task?.completionDelivery?.state, "claimed");
2451+
2452+
// The parent's Stop committed, but the restart cut the effect that stops its children.
2453+
const stoppedChild = yield* seedRestartCancelledChild({
2454+
parentThreadId: threadId,
2455+
projectId,
2456+
parentRunId: runId,
2457+
rootNodeId,
2458+
name: "restart-parent-stopped-child",
2459+
completionWake: "always",
2460+
continuationPending: true,
2461+
now,
2462+
});
2463+
yield* (yield* EventSink.EventSinkV2).write({
2464+
commandId: CommandId.make("command:restart-declined-parent:stop"),
2465+
events: [
2466+
{
2467+
id: EventId.make("event:restart-declined-parent:interrupt-request"),
2468+
type: "turn-item.updated",
2469+
threadId,
2470+
runId,
2471+
nodeId: rootNodeId,
2472+
providerInstanceId: modelSelection.instanceId,
2473+
occurredAt: now,
2474+
payload: {
2475+
id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"),
2476+
type: "run_interrupt_request",
2477+
threadId,
2478+
runId,
2479+
nodeId: rootNodeId,
2480+
providerThreadId: null,
2481+
providerTurnId: null,
2482+
nativeItemRef: null,
2483+
parentItemId: null,
2484+
ordinal: 90,
2485+
status: "completed",
2486+
title: "Interrupt requested",
2487+
startedAt: now,
2488+
completedAt: now,
2489+
updatedAt: now,
2490+
message: "Stopped by user",
2491+
},
2492+
},
2493+
],
2494+
});
2495+
yield* continueRestartedRun({
2496+
threadId: stoppedChild.childThreadId,
2497+
sourceRunId: stoppedChild.childRunId,
2498+
}).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })));
2499+
const afterStop = yield* orchestrator.getThreadProjection(threadId);
2500+
assert.equal(
2501+
afterStop.subagents.find((row) => row.id === stoppedChild.taskId)?.status,
2502+
"cancelled",
2503+
);
2504+
assert.deepEqual(
2505+
(yield* orchestrator.getThreadProjection(stoppedChild.childThreadId)).runs.map(
2506+
(run) => run.id,
2507+
),
2508+
[stoppedChild.childRunId],
2509+
);
24512510
}),
24522511
);
24532512

‎apps/server/src/orchestration-v2/Orchestrator.ts‎

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
107107
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
108108
import {
109109
cancelledBeforeStart,
110+
continuationFailedBeforePrompt,
111+
continuationStartRetryDelayMs,
110112
isRestartContinuationSource,
111113
runOutcome,
112114
} from "./RestartBackgroundNote.ts";
@@ -2209,6 +2211,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
22092211
),
22102212
);
22112213

2214+
// Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
2215+
const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
2216+
threadId: ThreadId,
2217+
runId: RunId,
2218+
) {
2219+
const projection = yield* projectionStore.getThreadRecords(
2220+
threadId,
2221+
["runs", "providerTurns", "turnItems"],
2222+
{ turnItemTypes: ["error"], turnItemRunIds: [runId] },
2223+
);
2224+
const run = projection.runs.find((candidate) => candidate.id === runId);
2225+
if (run === undefined) return;
2226+
const delayMs = continuationStartRetryDelayMs(run, projection);
2227+
if (delayMs === undefined) return;
2228+
const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
2229+
yield* writeSystemEvents(
2230+
[],
2231+
[
2232+
{
2233+
id: `effect:restart-continuation:${run.id}`,
2234+
commandId,
2235+
threadId,
2236+
request: { type: "provider-runtime.continue", sourceRunId: run.id },
2237+
availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
2238+
},
2239+
],
2240+
);
2241+
});
2242+
22122243
// A hold waits for a person to review what they queued. Nobody reviews messages from agents,
22132244
// the server or scheduled tasks, so a later completed run releases those unless it was stopped.
22142245
const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
@@ -5188,7 +5219,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
51885219
const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
51895220
if (
51905221
!source ||
5191-
!isRestartContinuationSource(source, projection.providerTurns) ||
5222+
!(
5223+
isRestartContinuationSource(source, projection.providerTurns) ||
5224+
continuationFailedBeforePrompt(source, projection.providerTurns)
5225+
) ||
51925226
projection.thread.archivedAt !== null ||
51935227
projection.thread.deletedAt !== null ||
51945228
projection.thread.providerInstanceId !== source.providerInstanceId ||
@@ -10419,7 +10453,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1041910453
// A resumed queue can have a lower ordinal than a pending continuation source.
1042010454
// Stop must reach those sources even when interrupting the resumed run succeeds.
1042110455
if (
10422-
run.status === "cancelled" &&
10456+
(run.status === "cancelled" ||
10457+
(run.status === "failed" && run.restartContinuationOfRunId !== undefined)) &&
1042310458
(yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
1042410459
) {
1042510460
yield* markStoppedRun({ ...input, thread, run });
@@ -13129,6 +13164,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1312913164
// while writing child events, so nesting parent inside child would invert the order.
1313013165
// The keyed semaphores are neither reentrant nor deadlock-aware.
1313113166
if (delegatedTaskTerminalStatus(run.status) !== null) {
13167+
// Before the parent hears of the failure: a pending continuation keeps the task open.
13168+
if (run.status === "failed" && run.restartContinuationOfRunId !== undefined) {
13169+
yield* threadDispatch.withLock(threadId, retryContinuationStart(threadId, run.id)).pipe(
13170+
Effect.catchCause((cause) =>
13171+
Effect.logWarning("Failed to schedule a restart continuation retry", {
13172+
threadId,
13173+
runId: run.id,
13174+
cause,
13175+
}),
13176+
),
13177+
);
13178+
}
1313213179
yield* threadDispatch.withLock(
1313313180
threadId,
1313413181
finalizeDelegatedCompletionDelivery(threadId, run.id),

‎apps/server/src/orchestration-v2/RestartBackgroundNote.ts‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import type {
66
OrchestrationV2RunAttempt,
77
OrchestrationV2TurnItem,
88
} from "@t3tools/contracts";
9-
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
9+
import { latestRootProviderFailure, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
1010

1111
type Work = OrchestrationV2RestartCancelledBackgroundWork;
1212
type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
@@ -181,6 +181,50 @@ export function cancelledBeforeStart(
181181
return run.status === "cancelled" && neverStarted(run);
182182
}
183183

184+
/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
185+
export function continuationFailedBeforePrompt(
186+
run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
187+
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
188+
): boolean {
189+
return (
190+
run.status === "failed" &&
191+
run.restartContinuationOfRunId !== undefined &&
192+
!providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
193+
);
194+
}
195+
196+
const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000];
197+
198+
/** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */
199+
export function continuationStartRetryDelayMs(
200+
run: OrchestrationV2Run,
201+
projection: {
202+
readonly runs: ReadonlyArray<OrchestrationV2Run>;
203+
readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
204+
readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
205+
},
206+
): number | undefined {
207+
if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return;
208+
// These fail the same way however often they start.
209+
const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class;
210+
if (
211+
failureClass === "usage_limit" ||
212+
failureClass === "permission_error" ||
213+
failureClass === "validation_error"
214+
)
215+
return;
216+
let failedStarts = 0;
217+
for (
218+
let current: OrchestrationV2Run | undefined = run;
219+
current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns);
220+
current = projection.runs.find(
221+
(candidate) => candidate.id === current?.restartContinuationOfRunId,
222+
)
223+
)
224+
failedStarts += 1;
225+
return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1];
226+
}
227+
184228
export function isRestartContinuationSource(
185229
source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
186230
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,

‎apps/server/src/orchestration-v2/RestartContinuation.test.ts‎

Lines changed: 120 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
2020
import * as Layer from "effect/Layer";
2121
import * as ServerSettings from "../serverSettings.ts";
2222
import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
23+
import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
2324
import * as ThreadManagementService from "./ThreadManagementService.ts";
2425
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
2526
import * as ProjectionStore from "./ProjectionStore.ts";
@@ -46,6 +47,8 @@ function makeProjection() {
4647
providerInstanceId: instanceId,
4748
archivedAt: null,
4849
deletedAt: null,
50+
lineage: { parentThreadId: null, relationshipToParent: null },
51+
forkedFrom: null,
4952
},
5053
runs: [
5154
{
@@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
839842
}),
840843
);
841844

842-
const continuationDispatches = (projection: OrchestrationV2ThreadProjection) =>
845+
const continuationDispatches = (
846+
projection: OrchestrationV2ThreadProjection,
847+
sourceRunId: RunId = runId,
848+
) =>
843849
Effect.gen(function* () {
844850
const dispatches: Array<Extract<OrchestrationV2Command, { type: "message.dispatch" }>> = [];
845-
yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
851+
yield* continueRestartedRun({ threadId, sourceRunId }).pipe(
846852
Effect.provide(
847853
Layer.merge(
848854
Layer.mock(ThreadManagementService.ThreadManagementService)({
@@ -943,6 +949,118 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
943949
}),
944950
);
945951

952+
it("backs off twice from a continuation the provider never took, then lets it fail", () => {
953+
const base = makeProjection();
954+
const failedStart = (id: string, of: RunId) => ({
955+
...base.runs[0]!,
956+
id: RunId.make(id),
957+
activeAttemptId: RunAttemptId.make(`attempt:${id}`),
958+
status: "failed" as const,
959+
restartContinuationOfRunId: of,
960+
});
961+
const first = failedStart("run:retry-1", runId);
962+
const second = failedStart("run:retry-2", first.id);
963+
const third = failedStart("run:retry-3", second.id);
964+
const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) =>
965+
({
966+
runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third],
967+
providerTurns,
968+
turnItems: [first, second, third].map((run) => ({
969+
id: `error:${run.id}`,
970+
type: "error",
971+
status: "failed",
972+
runId: run.id,
973+
nodeId: run.rootNodeId,
974+
ordinal: 1,
975+
updatedAt: now,
976+
failure: { class: failureClass, message: "Failed.", code: null, retryable: null },
977+
})),
978+
}) as unknown as Parameters<typeof continuationStartRetryDelayMs>[1];
979+
assert.deepEqual(
980+
[first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())),
981+
[60_000, 180_000, undefined],
982+
);
983+
for (const failureClass of ["usage_limit", "permission_error", "validation_error"])
984+
assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass)));
985+
// Once the provider took the prompt, starting again would repeat the turn.
986+
assert.isUndefined(
987+
continuationStartRetryDelayMs(
988+
first,
989+
projection("provider_error", [
990+
{ ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" },
991+
] as never),
992+
),
993+
);
994+
});
995+
996+
it.effect("continues again after a continuation fails before its prompt", () =>
997+
Effect.gen(function* () {
998+
const cut = cutMidTurn();
999+
const failedStart = {
1000+
...cut.runs[0]!,
1001+
id: RunId.make("run:failed-start"),
1002+
ordinal: 2,
1003+
activeAttemptId: RunAttemptId.make("attempt:failed-start"),
1004+
status: "failed",
1005+
restartContinuationOfRunId: runId,
1006+
};
1007+
const dispatches = yield* continuationDispatches(
1008+
{ ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection,
1009+
failedStart.id,
1010+
);
1011+
assert.deepEqual(
1012+
dispatches.map((command) => command.restartContinuationOfRunId),
1013+
[failedStart.id],
1014+
);
1015+
}),
1016+
);
1017+
1018+
it.effect.each([
1019+
["running", null, 0],
1020+
["completed", "Done.", 1],
1021+
] as const)(
1022+
"a parent's Stop on the task's run blocks the child only while the task is %s",
1023+
([status, result, expected]) =>
1024+
Effect.gen(function* () {
1025+
const parentThreadId = ThreadId.make("thread:parent");
1026+
const parentRunId = RunId.make("run:parent");
1027+
const taskId = NodeId.make("node:task");
1028+
const cut = cutMidTurn();
1029+
const child = {
1030+
...cut,
1031+
thread: {
1032+
...cut.thread,
1033+
lineage: { parentThreadId, relationshipToParent: "subagent" },
1034+
forkedFrom: { type: "node", nodeId: taskId },
1035+
},
1036+
} as unknown as OrchestrationV2ThreadProjection;
1037+
const parent = {
1038+
thread: { id: parentThreadId },
1039+
subagents: [{ id: taskId, runId: parentRunId, origin: "app_owned", status, result }],
1040+
turnItems: [{ id: "turn-item:stop", runId: parentRunId, type: "run_interrupt_request" }],
1041+
};
1042+
let dispatched = 0;
1043+
yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
1044+
Effect.provide(
1045+
Layer.merge(
1046+
Layer.mock(ThreadManagementService.ThreadManagementService)({
1047+
getThreadRecords: (id) =>
1048+
Effect.succeed((id === parentThreadId ? parent : child) as never),
1049+
recoverDelegatedTask: () => Effect.void,
1050+
dispatch: () =>
1051+
Effect.sync(() => {
1052+
dispatched += 1;
1053+
return {} as never;
1054+
}),
1055+
}),
1056+
ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1057+
),
1058+
),
1059+
);
1060+
assert.equal(dispatched, expected);
1061+
}),
1062+
);
1063+
9461064
it.effect("does not continue a run the user asked to stop before the restart", () =>
9471065
Effect.gen(function* () {
9481066
const texts = yield* continuationTexts({

0 commit comments

Comments
 (0)